From 83fc453e0afc23c1ae30b8d5494e608767dd6ade Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 09:15:05 -0800 Subject: [PATCH 01/39] Initial SA recipe database. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 38 + ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 22 + ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 22 + ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 26 + ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 22 + ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 22 + ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 22 + ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 22 + ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 22 + ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 22 + ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 26 + ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 26 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 22 + ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 26 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 22 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 22 + ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 26 + ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 22 + ...oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 24 + ...oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 24 + ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 24 + ...oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 23 + ...oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 23 + ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 23 + .../configure/database/scenario_list.yaml | 1512 +++++++++++++++++ 170 files changed, 5466 insertions(+) create mode 100644 tensorrt_llm/configure/database.py create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/scenario_list.yaml diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py new file mode 100644 index 000000000000..e093d17fde90 --- /dev/null +++ b/tensorrt_llm/configure/database.py @@ -0,0 +1,38 @@ +from pathlib import Path +from typing import List, Optional + +import pandas as pd +import yaml +from pydantic import BaseModel + +DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml" + + +class RecipeRecord(BaseModel): + model: str + gpu: str + isl: int + osl: int + concurrency: int + config_path: str + tps_per_user: Optional[int] = None + tps_per_gpu: Optional[int] = None + num_gpus: int + + @classmethod + def from_yaml(cls, yaml_path: Path) -> List["RecipeRecord"]: + with open(yaml_path, "r") as f: + data = yaml.load(f, Loader=yaml.FullLoader) + return [cls(**item) for item in data] + + def to_pandas_row(self) -> pd.Series: + return pd.Series(self.model_dump()) + + +def get_recipe_dataframe(yaml_path: Path) -> pd.DataFrame: + records = RecipeRecord.from_yaml(yaml_path) + return pd.DataFrame([record.to_pandas_row() for record in records]) + + +def load_recipe_database() -> pd.DataFrame: + return get_recipe_dataframe(DATABASE_LIST_PATH) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..4725c335a2fc --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..778dba5027c8 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..1724b0b8af4b --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..cc943a4e5ad9 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..f06e5f4b1d74 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..e31b303f7337 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..dbb06f58e467 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..f7eccec94125 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..d813f6fbf0d0 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..c70a7dd6c6ee --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 256 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..e6581080c029 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..0826c51771ef --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..39ecdca068ff --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..3b346249fc45 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..786ec7fe456c --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..a4e5eabe851a --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..932933361a74 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..ff7bffb1da18 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..dcc1392d2fd7 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..2f17c33de980 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 128 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml new file mode 100644 index 000000000000..a422e84c1955 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..18b25097ea28 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..1b8401a825f9 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..1b530b5f5f6e --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..551e8b25cb37 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..38b625426ba8 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..ec59a1bd5052 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml new file mode 100644 index 000000000000..deb3f13ec516 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..69c72d4dafb0 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml new file mode 100644 index 000000000000..d57c515f196b --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..320198425c91 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..e001ce83a9a7 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..c5c393bf1051 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..2242bc618914 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml new file mode 100644 index 000000000000..bf1a026cd2db --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..56fc955f2b9a --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..46ef51eb8e29 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..f29b6e2bd350 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..efdf16ff5e09 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..0041a26c747e --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..7151a5a2f33c --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml new file mode 100644 index 000000000000..705aefdc6602 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 128 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..adfdfb5cefb6 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml new file mode 100644 index 000000000000..fa01711d9df7 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 256 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..d650e3feb927 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..23e7cd45ac7f --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..99620affd547 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,26 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: true + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: CUTLASS + attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..dec1388712f7 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +scenario: + model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 512 + enable_attention_dp: false + print_iter_log: true + kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false + stream_interval: 10 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..f6ff78bcddae --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..13952e7762a5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..67c8cf3ee558 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..2eabed9bcf94 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..246decd0fd96 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..cfbb10e76771 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..fd9218e8ae32 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..b4991a90e933 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..9527b6f0bc30 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..b57df2745953 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..820fc7ae0db1 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..ea0821a0f86f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..70e6eb7210ff --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..c39784c47845 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..c1ef5051906c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..a521f7ada743 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..b9f33be7e853 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..d9f21c5bd1cd --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..1399bcd8d79f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..d26d7d09586c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml new file mode 100644 index 000000000000..24fe38a91547 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml new file mode 100644 index 000000000000..4aec4e67aa2c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml new file mode 100644 index 000000000000..828907414fb9 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml new file mode 100644 index 000000000000..c937c1790f4e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml new file mode 100644 index 000000000000..e0705dd791e6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml new file mode 100644 index 000000000000..189f9ffa2772 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml new file mode 100644 index 000000000000..86d778e36820 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml new file mode 100644 index 000000000000..d45e090b19e8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml new file mode 100644 index 000000000000..c7c6b64ae0ca --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml new file mode 100644 index 000000000000..d6db00bad77d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml new file mode 100644 index 000000000000..0c6894a4aa49 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml new file mode 100644 index 000000000000..cf83f8b20273 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml new file mode 100644 index 000000000000..c151d91c550c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml new file mode 100644 index 000000000000..b77e9a76702f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml new file mode 100644 index 000000000000..b225cbec3ef3 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml new file mode 100644 index 000000000000..12671456b7c8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml new file mode 100644 index 000000000000..1ac9a817c3d6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml new file mode 100644 index 000000000000..ad803d1b735b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml new file mode 100644 index 000000000000..0828c8c7b7a3 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml new file mode 100644 index 000000000000..14e8ce0e7487 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..1a641445db0f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..8235cb5cfcef --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..6eb6bc9c941a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..15642e38b5b0 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..6cfbf7fd3814 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..47c8a10c41af --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..b1e4c1e7a248 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..d800736568ec --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..231f46619ae8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..a2785728d660 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..da1eb7ba274f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..a1afef408818 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..fd7f6010bc42 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..817b4edd1f4a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..180b978f373c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..c325516359d0 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..a71283a4141f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..bb6bb88818b8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..03b5d975965b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..ea085d3b566a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,24 @@ +scenario: + model: openai/gpt-oss-120b + gpu: B200_NVL + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + print_iter_log: true + stream_interval: 20 + num_postprocess_workers: 4 + moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..a7f028528add --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..50ec01cf3d9e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..5095f52556a2 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..9ab79f6d4c8d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..ee96a94e02ae --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..386944f8df14 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..6fbcab4deb34 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..b5dc97dc1e87 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..296bc44a45c5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..04974c0ae5e4 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..208e378dfd26 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..1f4539c5f5f8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..0c7412cdcb61 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..83ca2b959214 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..917a5302ae5f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..fe202ea3a74a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..35ce1c525e21 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..090b845f106e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..45e3e3578e4c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..0a2c9f03e7e9 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml new file mode 100644 index 000000000000..c8dd741573f4 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml new file mode 100644 index 000000000000..ba5550457957 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml new file mode 100644 index 000000000000..2e65b4c58aed --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml new file mode 100644 index 000000000000..237fbcbca6de --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml new file mode 100644 index 000000000000..74f1eb81e185 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml new file mode 100644 index 000000000000..49806dc006cd --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml new file mode 100644 index 000000000000..8a20bacb3746 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml new file mode 100644 index 000000000000..781cf8c54629 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml new file mode 100644 index 000000000000..8c48d05d31f7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml new file mode 100644 index 000000000000..e384f100e5a2 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml new file mode 100644 index 000000000000..e7bfd019857c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml new file mode 100644 index 000000000000..f84382235406 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml new file mode 100644 index 000000000000..23adf309293a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml new file mode 100644 index 000000000000..2ccfba511de7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml new file mode 100644 index 000000000000..8150379dd83b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml new file mode 100644 index 000000000000..cab722643fc7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml new file mode 100644 index 000000000000..bdad6740434f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml new file mode 100644 index 000000000000..750c34944182 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml new file mode 100644 index 000000000000..bac10aff8956 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml new file mode 100644 index 000000000000..7bd32c84151a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 1024 + target_osl: 8192 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..d71b30b0a630 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..20fce080a34c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..487f189b4d1a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..9561df3add9a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..1e2ddb8bea41 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 1 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..378972ee0fb8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..a1dd4d0b8d83 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..76b5ac58593b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..8aafc181e1f1 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..c86014915bde --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 2 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..777747a68c37 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..14a0b5a8437c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..274c4a5657df --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..93ea732ea408 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..d47c769b604a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 4 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..0f48bb8e1439 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 16 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 16 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..98e6f59141ed --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 32 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 32 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..27238bb87641 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 4 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 4 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..663fbc3e3793 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 64 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 64 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..6e3f80882b7e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -0,0 +1,23 @@ +scenario: + model: openai/gpt-oss-120b + gpu: H200_SXM + num_gpus: 8 + target_isl: 8192 + target_osl: 1024 + target_concurrency: 8 +env: + TRTLLM_ENABLE_PDL: 1 +config: + cuda_graph_config: + enable_padding: true + max_batch_size: 8 + enable_attention_dp: false + kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 + moe_config: + backend: TRITON + num_postprocess_workers: 4 + print_iter_log: true + stream_interval: 20 diff --git a/tensorrt_llm/configure/database/scenario_list.yaml b/tensorrt_llm/configure/database/scenario_list.yaml new file mode 100644 index 000000000000..dacb09069ccb --- /dev/null +++ b/tensorrt_llm/configure/database/scenario_list.yaml @@ -0,0 +1,1512 @@ +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 128 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 256 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: nvidia/DeepSeek-R1-0528-FP4-v2 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: deepseek-ai/DeepSeek-R1-0528 + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: B200_NVL + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 1024 + osl: 8192 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 1 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 2 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 4 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 16 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 32 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 4 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 64 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 +- model: openai/gpt-oss-120b + gpu: H200_SXM + isl: 8192 + osl: 1024 + concurrency: 8 + config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml + tps_per_user: null + tps_per_gpu: null + num_gpus: 8 From 1b6cf14c4755a79736ebcdef9114a4d0f39480bd Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 18:43:31 -0800 Subject: [PATCH 02/39] Rename scenaro to constraints Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 2 +- 168 files changed, 168 insertions(+), 168 deletions(-) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml index 4725c335a2fc..c32260a8b365 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml index 778dba5027c8..3b0b031c53e4 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml index 1724b0b8af4b..e48e9516ab1a 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml index cc943a4e5ad9..d46158b4ef2e 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml index f06e5f4b1d74..dafe1f8d5985 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml index e31b303f7337..22c518dd0dc7 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml index dbb06f58e467..04875f5f354f 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml index f7eccec94125..9e8c7bf54bf5 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml index d813f6fbf0d0..717b211778ee 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml index c70a7dd6c6ee..ddeb7068476d 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml index e6581080c029..bb392c21ae2f 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml index 0826c51771ef..13c556051e85 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml index 39ecdca068ff..0379939797bd 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml index 3b346249fc45..897af3f38c18 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml index 786ec7fe456c..a06a7927aebc 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml index a4e5eabe851a..1de38df7ce56 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml index 932933361a74..6e805c2c9ae9 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml index ff7bffb1da18..586e24ad4a43 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml index dcc1392d2fd7..8023959ef8fc 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml index 2f17c33de980..d4c9c6a6f476 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml index a422e84c1955..eb803a65ce7a 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml index 18b25097ea28..ca329b80af8e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml index 1b8401a825f9..45efb8cf53a3 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml index 1b530b5f5f6e..24c49d146bd2 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml index 551e8b25cb37..3262f411d32d 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml index 38b625426ba8..bed3c77816df 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml index ec59a1bd5052..0a98ec0becc0 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml index deb3f13ec516..f69f3f777301 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml index 69c72d4dafb0..d9d64e3a389e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml index d57c515f196b..9e4eccd4971f 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml index 320198425c91..cc77ecb9ae06 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml index e001ce83a9a7..9ccb55f1f0a4 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml index c5c393bf1051..d99ed3d65a9d 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml index 2242bc618914..aab1a81302a9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml index bf1a026cd2db..0430562fa590 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml index 56fc955f2b9a..f0ce6a6a3531 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml index 46ef51eb8e29..8a82d06ab8ed 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml index f29b6e2bd350..d168277f2930 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml index efdf16ff5e09..189c6add635f 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml index 0041a26c747e..989dd9ff357d 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml index 7151a5a2f33c..4dbd12b3c63c 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml index 705aefdc6602..ce5975fbed4e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml index adfdfb5cefb6..7440d24f2603 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml index fa01711d9df7..63cb11dc39a0 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml index d650e3feb927..5162ce288808 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml index 23e7cd45ac7f..a54b9f135e5f 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml index 99620affd547..9089abde72b4 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml index dec1388712f7..a469fb03f243 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml index f6ff78bcddae..d5f0550ea1b0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml index 13952e7762a5..89b55714267c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml index 67c8cf3ee558..1613da6b48f8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml index 2eabed9bcf94..f6f9e8c3f649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml index 246decd0fd96..c82b6d942499 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml index cfbb10e76771..1ce17af3fff8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml index fd9218e8ae32..835418d65518 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml index b4991a90e933..f41a5509f55e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml index 9527b6f0bc30..5c46254934cd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml index b57df2745953..bf21a4008393 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml index 820fc7ae0db1..5f4e4f3d0917 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml index ea0821a0f86f..b308fae2b635 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml index 70e6eb7210ff..57415b1ac30e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml index c39784c47845..99a2dec59c77 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml index c1ef5051906c..929c29419173 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml index a521f7ada743..51b4c0ef4260 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml index b9f33be7e853..f21524912fc5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml index d9f21c5bd1cd..14a8731ccd71 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml index 1399bcd8d79f..0a95a868a2c7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml index d26d7d09586c..277c40c69638 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml index 24fe38a91547..ac4d6e378192 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml index 4aec4e67aa2c..82930e426934 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml index 828907414fb9..1c4f51de5e27 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml index c937c1790f4e..5fd46eedbec9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml index e0705dd791e6..efb4aa2b72a2 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml index 189f9ffa2772..bb19699c4e0b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml index 86d778e36820..97abd8b3b32e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml index d45e090b19e8..815c6f330dbc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml index c7c6b64ae0ca..0c66a691f4d2 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml index d6db00bad77d..530e113b7642 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml index 0c6894a4aa49..e81dc73007a9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml index cf83f8b20273..6039f66643c5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml index c151d91c550c..87cf5aee57e5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml index b77e9a76702f..ab86c90b0f94 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml index b225cbec3ef3..0d36f3d5b947 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml index 12671456b7c8..9c0b40f3f622 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml index 1ac9a817c3d6..41831f3b101e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml index ad803d1b735b..4d7c08e07ed0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml index 0828c8c7b7a3..454fb7f7d092 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml index 14e8ce0e7487..c4d8d751f9ef 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml index 1a641445db0f..9e4c63ad5244 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml index 8235cb5cfcef..c0f07ae30c2e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml index 6eb6bc9c941a..c63ce43af3ea 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml index 15642e38b5b0..2870aef58ce4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml index 6cfbf7fd3814..eeaed03658c1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml index 47c8a10c41af..4825fd4f104b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml index b1e4c1e7a248..6962e2cd5bb3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml index d800736568ec..f6fab914bf4f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml index 231f46619ae8..9728183b72ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml index a2785728d660..f72199de217c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml index da1eb7ba274f..f45b3ba06497 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml index a1afef408818..25152fe7193e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml index fd7f6010bc42..7e8b593ce809 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml index 817b4edd1f4a..1b6e1a065cc1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml index 180b978f373c..c8af20ef5ac5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml index c325516359d0..7988b5983161 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml index a71283a4141f..75c4072286cf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml index bb6bb88818b8..9bcef306ea24 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml index 03b5d975965b..1f651261c4bf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml index ea085d3b566a..d1a8583a35fd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: B200_NVL num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml index a7f028528add..88b0d04ccbc7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml index 50ec01cf3d9e..49edff89a581 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml index 5095f52556a2..efb12b2990cd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml index 9ab79f6d4c8d..97e5c74b6d96 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml index ee96a94e02ae..6ff750c395c1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml index 386944f8df14..e8297ab21eb4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml index 6fbcab4deb34..28f755db1b25 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml index b5dc97dc1e87..88713c60c6b9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml index 296bc44a45c5..5d037f31c48c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml index 04974c0ae5e4..412dc2caf305 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml index 208e378dfd26..1ae5fb2bd5a7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml index 1f4539c5f5f8..0be3059dc658 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml index 0c7412cdcb61..734dddd62f34 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml index 83ca2b959214..66993b96fa3f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml index 917a5302ae5f..e5a3d155cb97 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml index fe202ea3a74a..00025fe7e36b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml index 35ce1c525e21..0399f460e166 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml index 090b845f106e..6ba67335e3de 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml index 45e3e3578e4c..67829867b918 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml index 0a2c9f03e7e9..3f2c9b7ae225 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml index c8dd741573f4..f03a6a17d31f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml index ba5550457957..74cedf07a862 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml index 2e65b4c58aed..320cd4b678b5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml index 237fbcbca6de..d6f4b02c7743 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml index 74f1eb81e185..f23993712fec 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml index 49806dc006cd..f9f20e1bb8fe 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml index 8a20bacb3746..8894a2b85476 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml index 781cf8c54629..045cf268f942 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml index 8c48d05d31f7..1c386c7186f0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml index e384f100e5a2..da88c0116597 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml index e7bfd019857c..5bc5999fac4f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml index f84382235406..2d156af01d7e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml index 23adf309293a..452105fa700c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml index 2ccfba511de7..fc6d4b9357c9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml index 8150379dd83b..bebdda511b6a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml index cab722643fc7..97062cac5759 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml index bdad6740434f..b262cee2fc7c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml index 750c34944182..2799aaa47328 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml index bac10aff8956..b1f3f9a3e7d6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml index 7bd32c84151a..774005a2b9d4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml index d71b30b0a630..04bc59300e64 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml index 20fce080a34c..3ef38bbf1071 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml index 487f189b4d1a..1f7d8f55eeb0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml index 9561df3add9a..6eebb3597dfb 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml index 1e2ddb8bea41..83eed2725443 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 1 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml index 378972ee0fb8..42fce0a487c8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml index a1dd4d0b8d83..2da5debb90cc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml index 76b5ac58593b..4ba1c56d1411 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml index 8aafc181e1f1..56fe2397d584 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml index c86014915bde..d7f5958b5cbf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 2 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml index 777747a68c37..af86bbff53e0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml index 14a0b5a8437c..ef0df13921fd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml index 274c4a5657df..e8d20a5911ec 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml index 93ea732ea408..1e5a5b8abe51 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml index d47c769b604a..941ccfaaea97 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 4 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml index 0f48bb8e1439..b265a2be47f0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml index 98e6f59141ed..84f5d2664499 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml index 27238bb87641..1f2eeb238d55 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml index 663fbc3e3793..3467cc360761 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml index 6e3f80882b7e..de54bc31d23c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -1,4 +1,4 @@ -scenario: +constraints: model: openai/gpt-oss-120b gpu: H200_SXM num_gpus: 8 From d8597092010437b0e4fff09e04a5261d70980b0a Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 19:59:41 -0800 Subject: [PATCH 03/39] Port over to RootModel Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 71 +++++++++++++++++++++--------- 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index e093d17fde90..74e1fff7e2ba 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -1,38 +1,69 @@ from pathlib import Path -from typing import List, Optional +from typing import Iterator, List, Optional import pandas as pd import yaml -from pydantic import BaseModel +from pydantic import BaseModel, Field, RootModel DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml" class RecipeRecord(BaseModel): - model: str - gpu: str - isl: int - osl: int - concurrency: int - config_path: str - tps_per_user: Optional[int] = None - tps_per_gpu: Optional[int] = None - num_gpus: int + model: str = Field(description="Model name") + gpu: str = Field(description="GPU name") + isl: int = Field(description="Input sequence length") + osl: int = Field(description="Output sequence length") + concurrency: int = Field(description="Concurrency") + config_path: str = Field(description="Configuration path") + tps_per_user: Optional[int] = Field(description="TPS per user", default=None) + tps_per_gpu: Optional[int] = Field(description="TPS per GPU", default=None) + num_gpus: int = Field(description="Number of GPUs") + def to_pandas_row(self) -> pd.Series: + """Convert recipe record to pandas Series. + + Returns: + pd.Series: Pandas Series containing the recipe record. + """ + return pd.Series(self.model_dump()) + + +class RecipeList(RootModel[List[RecipeRecord]]): @classmethod - def from_yaml(cls, yaml_path: Path) -> List["RecipeRecord"]: + def from_yaml(cls, yaml_path: Path) -> "RecipeList": + """Load recipe list from YAML file and validate the data. + + Args: + yaml_path (Path): Path to the YAML file containing the recipe list. + + Returns: + RecipeList: Recipe list object. + """ with open(yaml_path, "r") as f: data = yaml.load(f, Loader=yaml.FullLoader) - return [cls(**item) for item in data] + return cls(data) - def to_pandas_row(self) -> pd.Series: - return pd.Series(self.model_dump()) + def to_pandas_dataframe(self) -> pd.DataFrame: + """Convert recipe list to pandas DataFrame. + + Returns: + pd.DataFrame: Pandas DataFrame where each row contains a recipe record. + """ + return pd.DataFrame([record.to_pandas_row() for record in self.root]) + def __iter__(self) -> Iterator[RecipeRecord]: + """Iterate over the recipe list. -def get_recipe_dataframe(yaml_path: Path) -> pd.DataFrame: - records = RecipeRecord.from_yaml(yaml_path) - return pd.DataFrame([record.to_pandas_row() for record in records]) + Returns: + Iterator[RecipeRecord]: Iterator over the recipe list. + """ + for record in self.root: + yield record + def __len__(self) -> int: + """Get the number of recipes in the list. -def load_recipe_database() -> pd.DataFrame: - return get_recipe_dataframe(DATABASE_LIST_PATH) + Returns: + int: Number of recipes in the list. + """ + return len(self.root) From 286bb04d4d78605766e208069496be59f1408604 Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:16:39 -0800 Subject: [PATCH 04/39] Update recipe classes. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 74e1fff7e2ba..9582e801ea54 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Iterator, List, Optional +from typing import Any, Dict, Iterator, List, Optional import pandas as pd import yaml @@ -8,7 +8,9 @@ DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml" -class RecipeRecord(BaseModel): +class RecipeConstraints(BaseModel): + """Recipe record for scenario list.""" + model: str = Field(description="Model name") gpu: str = Field(description="GPU name") isl: int = Field(description="Input sequence length") @@ -28,7 +30,15 @@ def to_pandas_row(self) -> pd.Series: return pd.Series(self.model_dump()) -class RecipeList(RootModel[List[RecipeRecord]]): +class Recipe(BaseModel): + """Recipe that describes a single scenario.""" + + constraints: RecipeConstraints = Field(description="Recipe constraints") + env_overrides: Dict[str, Any] = Field(description="Environment overrides", default_factory=dict) + config: Dict[str, Any] = Field(description="Configuration overrides", default_factory=dict) + + +class RecipeList(RootModel[List[RecipeConstraints]]): @classmethod def from_yaml(cls, yaml_path: Path) -> "RecipeList": """Load recipe list from YAML file and validate the data. @@ -51,7 +61,7 @@ def to_pandas_dataframe(self) -> pd.DataFrame: """ return pd.DataFrame([record.to_pandas_row() for record in self.root]) - def __iter__(self) -> Iterator[RecipeRecord]: + def __iter__(self) -> Iterator[RecipeConstraints]: """Iterate over the recipe list. Returns: From 066afb84cda81c00f0ac0963ee311be78e140310 Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:27:20 -0800 Subject: [PATCH 05/39] Update env to env_overrides. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 3 ++- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 2 +- .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 2 +- .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 2 +- 168 files changed, 169 insertions(+), 168 deletions(-) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml index c32260a8b365..bc11985520fa 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml index 3b0b031c53e4..007485c83102 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml index e48e9516ab1a..cfad1d587d33 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,8 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: + +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml index d46158b4ef2e..77977855bdd7 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml index dafe1f8d5985..dcd3dc25ca71 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml index 22c518dd0dc7..f01f36528e0b 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml index 04875f5f354f..c1ffca06dfd1 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml index 9e8c7bf54bf5..4f3f547628d3 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml index 717b211778ee..e0308d9c4b6d 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml index ddeb7068476d..fee6bb4c0bc8 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml index bb392c21ae2f..e9b11f0b9865 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml index 13c556051e85..ece060c8d50e 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml index 0379939797bd..f87be9b97437 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml index 897af3f38c18..42c88d66c5e3 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml index a06a7927aebc..9abe8950140b 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml index 1de38df7ce56..8045307da664 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml index 6e805c2c9ae9..d90b6207bf05 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml index 586e24ad4a43..ef58a2020a02 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml index 8023959ef8fc..891784943e1f 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml index d4c9c6a6f476..8fd9790fca3e 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml index eb803a65ce7a..1ce823ebce33 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml index ca329b80af8e..eb9f9e67fc40 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml index 45efb8cf53a3..2f0bd54767f1 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml index 24c49d146bd2..a6bca1b4f8ec 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml index 3262f411d32d..6daee6414106 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml index bed3c77816df..618fe65a1edd 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml index 0a98ec0becc0..454ac155fd0e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml index f69f3f777301..e4fd062546eb 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml index d9d64e3a389e..1ca81b3f047a 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml index 9e4eccd4971f..fa9adde1fc73 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml index cc77ecb9ae06..622934a54543 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml index 9ccb55f1f0a4..962fab4f8ff3 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml index d99ed3d65a9d..2fbff7bd5932 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml index aab1a81302a9..754ecd630e34 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml index 0430562fa590..696a10ede199 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml index f0ce6a6a3531..336ecb1375de 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml index 8a82d06ab8ed..9eeb9fe78fd2 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml index d168277f2930..38518b790dc9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml index 189c6add635f..5d3a01c77a10 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml index 989dd9ff357d..1c6ca3c7d03c 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml index 4dbd12b3c63c..e880f71dacb9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml index ce5975fbed4e..08ae474cb1c9 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 128 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml index 7440d24f2603..ed8dc45b999b 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml index 63cb11dc39a0..db542084c7ac 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 256 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml index 5162ce288808..2398a785a9c6 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml index a54b9f135e5f..39797e4fd262 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml index 9089abde72b4..51e4cf3be355 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml index a469fb03f243..6f6ac3c4efbc 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml index d5f0550ea1b0..07ab42c83dc7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml index 89b55714267c..0bc68ec73e0b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml index 1613da6b48f8..0e7cdcb0fed0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml index f6f9e8c3f649..5b33b0061109 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml index c82b6d942499..ddde04632fef 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml index 1ce17af3fff8..e2423f02b31b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml index 835418d65518..5eb831f7b771 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml index f41a5509f55e..25af6251f57f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml index 5c46254934cd..27ea9bdc2c0c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml index bf21a4008393..151c830b7114 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml index 5f4e4f3d0917..34c577437f6d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml index b308fae2b635..cf36077532f4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml index 57415b1ac30e..a67b9048aacc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml index 99a2dec59c77..5c03bc1288b5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml index 929c29419173..08d62899be07 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml index 51b4c0ef4260..139b661d9523 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml index f21524912fc5..c528c248e40e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml index 14a8731ccd71..8e64d3238545 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml index 0a95a868a2c7..2102a649c6d8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml index 277c40c69638..638e74ce66f9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml index ac4d6e378192..a1f54c1bed4c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml index 82930e426934..c2941b4776c4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml index 1c4f51de5e27..5d22acc2b3ef 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml index 5fd46eedbec9..2a23217ff87c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml index efb4aa2b72a2..b9b567e3c6cf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml index bb19699c4e0b..c103c1c8d2bc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml index 97abd8b3b32e..544adfd99e38 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml index 815c6f330dbc..188bf67cbb2c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml index 0c66a691f4d2..8cb27e0a60c3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml index 530e113b7642..7682c3aae59b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml index e81dc73007a9..6b8a5c99a24d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml index 6039f66643c5..29e7dd016023 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml index 87cf5aee57e5..81c963bc3d66 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml index ab86c90b0f94..d43af26ceb14 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml index 0d36f3d5b947..70a2a883cbd1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml index 9c0b40f3f622..087811f62ea7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml index 41831f3b101e..9dd73aa1bc65 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml index 4d7c08e07ed0..c5d51031dc7a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml index 454fb7f7d092..9f67d733f83d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml index c4d8d751f9ef..b9fa200f76fa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml index 9e4c63ad5244..cc70292eba00 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml index c0f07ae30c2e..c0c2048c2e26 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml index c63ce43af3ea..734f90ab3239 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml index 2870aef58ce4..b1bf229643e8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml index eeaed03658c1..1c0a5181dcac 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml index 4825fd4f104b..0305e02939e7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml index 6962e2cd5bb3..0753dfb129a4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml index f6fab914bf4f..5b4e0deaedd7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml index 9728183b72ca..d40f00d6f983 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml index f72199de217c..6f8e8db1458c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml index f45b3ba06497..cb2e735be596 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml index 25152fe7193e..bb6a98714c40 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml index 7e8b593ce809..45f749c8fcf9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml index 1b6e1a065cc1..16915e230b1b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml index c8af20ef5ac5..a0043a10f702 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml index 7988b5983161..4b5eac7996ff 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml index 75c4072286cf..fa2cac24e6fe 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml index 9bcef306ea24..910d9b621f85 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml index 1f651261c4bf..74802561ad1d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml index d1a8583a35fd..c3ac98310da5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml index 88b0d04ccbc7..19ed66323e75 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml index 49edff89a581..cf38aa9547d6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml index efb12b2990cd..1584f4231915 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml index 97e5c74b6d96..59f7e59a0c66 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml index 6ff750c395c1..bebec6d64c29 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml index e8297ab21eb4..89b43993793c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml index 28f755db1b25..ff90474ca631 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml index 88713c60c6b9..6b0f346b957e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml index 5d037f31c48c..7639a842953a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml index 412dc2caf305..055050456b38 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml index 1ae5fb2bd5a7..da7decb5c24c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml index 0be3059dc658..74bdd4ec6941 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml index 734dddd62f34..ba10fa27f660 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml index 66993b96fa3f..75560253a0ed 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml index e5a3d155cb97..26706dc2a9a3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml index 00025fe7e36b..710c4e7f3d08 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml index 0399f460e166..be72abced5e5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml index 6ba67335e3de..bd53e3a2251a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml index 67829867b918..4b8e06cd73d0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml index 3f2c9b7ae225..8b277eb06773 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml index f03a6a17d31f..d8b2d4cb7111 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml index 74cedf07a862..a909b624ecad 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml index 320cd4b678b5..acfc7da97178 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml index d6f4b02c7743..c1f8c66f9ca0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml index f23993712fec..574634491b44 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml index f9f20e1bb8fe..5954a22ea253 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml index 8894a2b85476..b5e20ed3f0a1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml index 045cf268f942..c23e61e524c1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml index 1c386c7186f0..b3d6f3522cda 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml index da88c0116597..7689682459e8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml index 5bc5999fac4f..09799ac87bd3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml index 2d156af01d7e..ee43cfa08998 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml index 452105fa700c..5d2f9278deae 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml index fc6d4b9357c9..cb6b902eae03 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml index bebdda511b6a..cda2dbd4be30 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml index 97062cac5759..a56724079357 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml index b262cee2fc7c..3abcf5b7f657 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml index 2799aaa47328..cfe49a3f53fc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml index b1f3f9a3e7d6..1dbd831b09b3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml index 774005a2b9d4..fa3ac5f75756 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 1024 target_osl: 8192 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml index 04bc59300e64..51e7db06098c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml index 3ef38bbf1071..20c70f3ad997 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml index 1f7d8f55eeb0..3b31277ea799 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml index 6eebb3597dfb..7b48e384c981 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml index 83eed2725443..4ba80525301e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml index 42fce0a487c8..951ca48e5217 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml index 2da5debb90cc..16e05ba27766 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml index 4ba1c56d1411..e759749e4eb8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml index 56fe2397d584..a3ef256793ce 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml index d7f5958b5cbf..45db5ad5dc9b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml index af86bbff53e0..3d751ee390da 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml index ef0df13921fd..165df6c51b99 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml index e8d20a5911ec..ed9ac653da27 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml index 1e5a5b8abe51..e8ea41b65258 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml index 941ccfaaea97..6c58809f77d7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml index b265a2be47f0..32052150a463 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 16 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml index 84f5d2664499..9c9115251f1d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 32 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml index 1f2eeb238d55..e175fadc5ca9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 4 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml index 3467cc360761..f1bacc77c3fa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 64 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml index de54bc31d23c..fa76ed2f6111 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -5,7 +5,7 @@ constraints: target_isl: 8192 target_osl: 1024 target_concurrency: 8 -env: +env_overrides: TRTLLM_ENABLE_PDL: 1 config: cuda_graph_config: From 2b059c2a28d330d1ffa7470cad6be1a5a0f19d3d Mon Sep 17 00:00:00 2001 From: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> Date: Tue, 18 Nov 2025 22:27:48 -0800 Subject: [PATCH 06/39] Update to add config recipe + load. Signed-off-by: Frank Di Natale <3429989+FrankD412@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 9582e801ea54..47655e2a5f87 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -29,6 +29,11 @@ def to_pandas_row(self) -> pd.Series: """ return pd.Series(self.model_dump()) + def load_config(self) -> Dict[str, Any]: + """Load configuration from the configuration path.""" + with open(self.config_path, "r") as f: + return Recipe(**yaml.load(f, Loader=yaml.FullLoader)) + class Recipe(BaseModel): """Recipe that describes a single scenario.""" From 84f5340a950171361dd31ac25a0373b3efe5e134 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 09:56:22 -0800 Subject: [PATCH 07/39] include recipe-db in packaging Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- setup.py | 1 + tensorrt_llm/configure/__init__.py | 0 2 files changed, 1 insertion(+) create mode 100644 tensorrt_llm/configure/__init__.py diff --git a/setup.py b/setup.py index 5c61029aad85..b368bfd082d1 100644 --- a/setup.py +++ b/setup.py @@ -132,6 +132,7 @@ def has_ext_modules(self): 'bench/build/benchmark_config.yml', 'evaluate/lm_eval_tasks/**/*', "_torch/auto_deploy/config/*.yaml", + "configure/database/**/*.yaml", # Include CUDA source for fused MoE align extension so runtime JIT can find it in wheels '_torch/auto_deploy/custom_ops/fused_moe/moe_align_kernel.cu', '_torch/auto_deploy/custom_ops/fused_moe/triton_fused_moe_configs/*' diff --git a/tensorrt_llm/configure/__init__.py b/tensorrt_llm/configure/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 From a5959243670fd64d8f2e8906a612dd3de6dd09b1 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 10:55:47 -0800 Subject: [PATCH 08/39] initial table generation artifacts Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- comprehensive_table.rst | 1356 ++++++++++++++++++++++++ docs/source/deployment-guide/index.rst | 7 + tools/generate_config_table.py | 124 +++ 3 files changed, 1487 insertions(+) create mode 100644 comprehensive_table.rst create mode 100644 tools/generate_config_table.py diff --git a/comprehensive_table.rst b/comprehensive_table.rst new file mode 100644 index 000000000000..f4c66e0a0963 --- /dev/null +++ b/comprehensive_table.rst @@ -0,0 +1,1356 @@ +.. list-table:: + :header-rows: 1 + :widths: 15 10 12 6 6 6 20 25 + + * - Model Name + - GPU + - Performance Profile + - ISL + - OSL + - Concurrency + - Config + - Command + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 1024 + - 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 1024 + - 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 8192 + - 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 8192 + - 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 + - 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 + - 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 + - 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 + - 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 + - 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 + - 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` diff --git a/docs/source/deployment-guide/index.rst b/docs/source/deployment-guide/index.rst index ed7fd9c536ef..e43f83612a78 100644 --- a/docs/source/deployment-guide/index.rst +++ b/docs/source/deployment-guide/index.rst @@ -94,3 +94,10 @@ The deployment guides below provide more detailed instructions for serving speci deployment-guide-for-qwen3-on-trtllm.md deployment-guide-for-qwen3-next-on-trtllm.md deployment-guide-for-kimi-k2-thinking-on-trtllm.md + +Comprehensive Configuration Database +------------------------------------ + +The table below lists all available pre-configured model scenarios in the TensorRT-LLM configuration database. Each row represents a specific model, GPU, and performance profile combination with recommended request settings. + +.. include:: comprehensive_table.rst diff --git a/tools/generate_config_table.py b/tools/generate_config_table.py new file mode 100644 index 000000000000..3e1947ba4a5c --- /dev/null +++ b/tools/generate_config_table.py @@ -0,0 +1,124 @@ +import os +import sys +from collections import defaultdict + +import yaml + + +def generate_rst(yaml_path, output_file=None): + """Generate RST table from YAML config database. + + Args: + yaml_path: Path to scenario_list.yaml + output_file: Optional output file path. If None, prints to stdout. + """ + with open(yaml_path, "r") as f: + data = yaml.safe_load(f) + + # Group by key attributes to determine min/max concurrency + # Key: (model, gpu, isl, osl) -> list of entry + groups = defaultdict(list) + for entry in data: + key = (entry.get("model"), entry.get("gpu"), entry.get("isl"), entry.get("osl")) + groups[key].append(entry) + + # Prepare output lines + lines = [] + lines.append(".. list-table::") + lines.append(" :header-rows: 1") + lines.append(" :widths: 20 12 12 15 10 20 25") + lines.append("") + lines.append(" * - Model Name") + lines.append(" - GPU") + lines.append(" - Performance Profile") + lines.append(" - ISL / OSL") + lines.append(" - Concurrency") + lines.append(" - Config") + lines.append(" - Command") + + # Sort keys for consistent output + # Key elements might be None, so convert to str/int safely + sorted_keys = sorted( + groups.keys(), key=lambda k: (str(k[0]), str(k[1]), int(k[2] or 0), int(k[3] or 0)) + ) + + for key in sorted_keys: + entries = groups[key] + # Sort by concurrency + entries.sort(key=lambda x: int(x.get("concurrency", 0))) + + # Get concurrency range for this group + min_conc = int(entries[0].get("concurrency", 0)) + max_conc = int(entries[-1].get("concurrency", 0)) + conc_range = max_conc - min_conc + + for entry in entries: + model = entry.get("model", "N/A") + gpu = entry.get("gpu", "N/A") + isl = entry.get("isl", "N/A") + osl = entry.get("osl", "N/A") + conc = int(entry.get("concurrency", 0)) + config_path = entry.get("config_path", "") + + # Determine profile based on relative position in concurrency range + if len(entries) == 1: + # Single entry: use concurrency value as heuristic + # Lower concurrency typically means lower latency + if conc <= 16: + profile = "Min Latency" + elif conc >= 64: + profile = "Max Throughput" + else: + profile = "Balanced" + else: + # Multiple entries: use relative position + relative_pos = (conc - min_conc) / conc_range if conc_range > 0 else 0.5 + + if relative_pos < 0.33: + profile = "Min Latency" + elif relative_pos > 0.67: + profile = "Max Throughput" + else: + profile = "Balanced" + + full_config_path = os.path.join("tensorrt_llm/configure", config_path) + command = f"trtllm-serve {model} --extra_llm_api_options {full_config_path}" + + lines.append(f" * - {model}") + lines.append(f" - {gpu}") + lines.append(f" - {profile}") + lines.append(f" - {isl} / {osl}") + lines.append(f" - {conc}") + lines.append(f" - {full_config_path}") + lines.append(f" - ``{command}``") + + # Output to file or stdout + output_text = "\n".join(lines) + if output_file: + with open(output_file, "w") as f: + f.write(output_text) + print(f"Generated table written to: {output_file}", file=sys.stderr) + else: + print(output_text) + + +if __name__ == "__main__": + # Assume script is run from repo root or tools dir + script_dir = os.path.dirname(os.path.abspath(__file__)) + # script is in tools/, repo root is parent + repo_root = os.path.dirname(script_dir) + + # Just to be safe, look for the file in fixed location relative to this script + yaml_path = os.path.join(repo_root, "tensorrt_llm/configure/database/scenario_list.yaml") + + if not os.path.exists(yaml_path): + # Try relative to CWD if script logic above fails (e.g. symlinks or strange structure) + yaml_path = "tensorrt_llm/configure/database/scenario_list.yaml" + + if not os.path.exists(yaml_path): + print(f"Error: YAML file not found at {yaml_path}", file=sys.stderr) + sys.exit(1) + + # Generate to separate file + output_path = os.path.join(repo_root, "docs/source/deployment-guide/comprehensive_table.rst") + generate_rst(yaml_path, output_file=output_path) From da7c764aab2372635b80c561c279816af55819c1 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 11:43:17 -0800 Subject: [PATCH 09/39] flatten yaml to align schema Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 34 ++++++--------- ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 34 ++++++--------- ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 35 ++++++---------- ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 34 ++++++--------- ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 34 ++++++--------- ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 34 ++++++--------- ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 34 ++++++--------- ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 34 ++++++--------- ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 42 +++++++------------ ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 34 ++++++--------- ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 34 ++++++--------- ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 34 ++++++--------- ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 34 ++++++--------- ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 34 ++++++--------- ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 34 ++++++--------- ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 34 ++++++--------- ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 34 ++++++--------- ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 34 ++++++--------- ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 42 +++++++------------ ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 34 ++++++--------- ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 42 +++++++------------ ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 34 ++++++--------- ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 34 ++++++--------- ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 34 ++++++--------- ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 34 ++++++--------- ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 42 +++++++------------ ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 34 ++++++--------- ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 34 ++++++--------- ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 34 ++++++--------- ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 42 +++++++------------ ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 42 +++++++------------ ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 34 ++++++--------- ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 34 ++++++--------- ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 42 +++++++------------ ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 42 +++++++------------ ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 34 ++++++--------- ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 42 +++++++------------ ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 34 ++++++--------- ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 34 ++++++--------- ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 42 +++++++------------ ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 34 ++++++--------- ...oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 34 ++++++--------- ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 34 ++++++--------- 168 files changed, 2176 insertions(+), 3617 deletions(-) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml index bc11985520fa..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml index 007485c83102..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml index cfad1d587d33..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -1,23 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 - -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml index 77977855bdd7..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml index dcd3dc25ca71..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml index f01f36528e0b..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml index c1ffca06dfd1..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml index 4f3f547628d3..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml index e0308d9c4b6d..87ec25cb384b 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -1,26 +1,16 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml index fee6bb4c0bc8..cea36102a893 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 256 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: DEEPGEMM +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml index e9b11f0b9865..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml index ece060c8d50e..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml index f87be9b97437..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml index 42c88d66c5e3..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml index 9abe8950140b..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml index 8045307da664..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml index d90b6207bf05..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml index ef58a2020a02..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml index 891784943e1f..267ee4500f9c 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -1,26 +1,16 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml index 8fd9790fca3e..c10047cbd6be 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: deepseek-ai/DeepSeek-R1-0528 - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 128 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml index 1ce823ebce33..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 128 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml index eb9f9e67fc40..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml index 2f0bd54767f1..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 256 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml index a6bca1b4f8ec..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml index 6daee6414106..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml index 618fe65a1edd..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml index 454ac155fd0e..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml index e4fd062546eb..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 128 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml index 1ca81b3f047a..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml index fa9adde1fc73..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 256 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml index 622934a54543..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml index 962fab4f8ff3..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml index 2fbff7bd5932..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml index 754ecd630e34..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml index 696a10ede199..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 128 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml index 336ecb1375de..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml index 9eeb9fe78fd2..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 256 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml index 38518b790dc9..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml index 5d3a01c77a10..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml index 1c6ca3c7d03c..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml index e880f71dacb9..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml index 08ae474cb1c9..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 128 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml index ed8dc45b999b..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml index db542084c7ac..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 256 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml index 2398a785a9c6..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml index 39797e4fd262..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml index 51e4cf3be355..33f5153ce191 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -1,26 +1,16 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: true - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: CUTLASS - attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml index 6f6ac3c4efbc..fa392eff1197 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -1,22 +1,12 @@ -constraints: - model: nvidia/DeepSeek-R1-0528-FP4-v2 - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 -env_overrides: - -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 512 - enable_attention_dp: false - print_iter_log: true - kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false - stream_interval: 10 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml index 07ab42c83dc7..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml index 0bc68ec73e0b..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml index 0e7cdcb0fed0..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml index 5b33b0061109..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml index ddde04632fef..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml index e2423f02b31b..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml index 5eb831f7b771..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml index 25af6251f57f..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml index 27ea9bdc2c0c..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml index 151c830b7114..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml index 34c577437f6d..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml index cf36077532f4..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml index a67b9048aacc..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml index 5c03bc1288b5..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml index 08d62899be07..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml index 139b661d9523..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml index c528c248e40e..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml index 8e64d3238545..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml index 2102a649c6d8..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml index 638e74ce66f9..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml index a1f54c1bed4c..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml index c2941b4776c4..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml index 5d22acc2b3ef..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml index 2a23217ff87c..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml index b9b567e3c6cf..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml index c103c1c8d2bc..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml index 544adfd99e38..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml index 188bf67cbb2c..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml index 8cb27e0a60c3..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml index 7682c3aae59b..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml index 6b8a5c99a24d..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml index 29e7dd016023..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml index 81c963bc3d66..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml index d43af26ceb14..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml index 70a2a883cbd1..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml index 087811f62ea7..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml index 9dd73aa1bc65..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml index c5d51031dc7a..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml index 9f67d733f83d..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml index b9fa200f76fa..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml index cc70292eba00..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml index c0c2048c2e26..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml index 734f90ab3239..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml index b1bf229643e8..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml index 1c0a5181dcac..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml index 0305e02939e7..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml index 0753dfb129a4..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml index 5b4e0deaedd7..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml index d40f00d6f983..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml index 6f8e8db1458c..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml index cb2e735be596..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml index bb6a98714c40..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml index 45f749c8fcf9..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml index 16915e230b1b..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml index a0043a10f702..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml index 4b5eac7996ff..2a5db8ffba68 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml index fa2cac24e6fe..93aa390c8825 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml index 910d9b621f85..fa26d3b12649 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml index 74802561ad1d..46baa7be7143 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml index c3ac98310da5..59cb144ec424 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -1,24 +1,16 @@ -constraints: - model: openai/gpt-oss-120b - gpu: B200_NVL - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 NCCL_GRAPH_REGISTER: 0 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - print_iter_log: true - stream_interval: 20 - num_postprocess_workers: 4 - moe_config: - backend: TRTLLM +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml index 19ed66323e75..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml index cf38aa9547d6..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml index 1584f4231915..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml index 59f7e59a0c66..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml index bebec6d64c29..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml index 89b43993793c..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml index ff90474ca631..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml index 6b0f346b957e..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml index 7639a842953a..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml index 055050456b38..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml index da7decb5c24c..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml index 74bdd4ec6941..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml index ba10fa27f660..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml index 75560253a0ed..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml index 26706dc2a9a3..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml index 710c4e7f3d08..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml index be72abced5e5..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml index bd53e3a2251a..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml index 4b8e06cd73d0..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml index 8b277eb06773..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml index d8b2d4cb7111..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml index a909b624ecad..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml index acfc7da97178..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml index c1f8c66f9ca0..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml index 574634491b44..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml index 5954a22ea253..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml index b5e20ed3f0a1..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml index c23e61e524c1..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml index b3d6f3522cda..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml index 7689682459e8..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml index 09799ac87bd3..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml index ee43cfa08998..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml index 5d2f9278deae..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml index cb6b902eae03..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml index cda2dbd4be30..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml index a56724079357..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml index 3abcf5b7f657..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml index cfe49a3f53fc..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml index 1dbd831b09b3..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml index fa3ac5f75756..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 1024 - target_osl: 8192 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml index 51e7db06098c..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml index 20c70f3ad997..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml index 3b31277ea799..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml index 7b48e384c981..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml index 4ba80525301e..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 1 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml index 951ca48e5217..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml index 16e05ba27766..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml index e759749e4eb8..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml index a3ef256793ce..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml index 45db5ad5dc9b..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 2 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml index 3d751ee390da..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml index 165df6c51b99..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml index ed9ac653da27..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml index e8ea41b65258..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml index 6c58809f77d7..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 4 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml index 32052150a463..e797d8e71c9a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 16 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 16 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml index 9c9115251f1d..1121d4e4b232 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 32 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 32 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml index e175fadc5ca9..2a74eaeccfaa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 4 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 4 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml index f1bacc77c3fa..e8c2ded159ca 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 64 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 64 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml index fa76ed2f6111..aa6194fedb46 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -1,23 +1,15 @@ -constraints: - model: openai/gpt-oss-120b - gpu: H200_SXM - num_gpus: 8 - target_isl: 8192 - target_osl: 1024 - target_concurrency: 8 env_overrides: TRTLLM_ENABLE_PDL: 1 -config: - cuda_graph_config: - enable_padding: true - max_batch_size: 8 - enable_attention_dp: false - kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 - moe_config: - backend: TRITON - num_postprocess_workers: 4 - print_iter_log: true - stream_interval: 20 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 From 3c4e0006557f850103ee703d4ae3883d5862119a Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 12:36:50 -0800 Subject: [PATCH 10/39] add comprehensive table Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .gitignore | 1 + .../deployment-guide/comprehensive_table.rst | 1187 +++++++++++++++++ 2 files changed, 1188 insertions(+) create mode 100644 docs/source/deployment-guide/comprehensive_table.rst diff --git a/.gitignore b/.gitignore index 78d8da20e478..bf69cca9b787 100644 --- a/.gitignore +++ b/.gitignore @@ -55,6 +55,7 @@ tensorrt_llm/scripts *docs/source/_cpp_gen* docs/source/**/*.rst !docs/source/examples/index.rst +!docs/source/deployment-guide/comprehensive_table.rst *.swp # Testing diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst new file mode 100644 index 000000000000..875b92bb85a6 --- /dev/null +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -0,0 +1,1187 @@ +.. list-table:: + :header-rows: 1 + :widths: 20 12 12 15 10 20 25 + + * - Model Name + - GPU + - Performance Profile + - ISL / OSL + - Concurrency + - Config + - Command + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - B200_NVL + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` + * - deepseek-ai/DeepSeek-R1-0528 + - H200_SXM + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 / 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 1024 / 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 1024 / 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 1024 / 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Min Latency + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 / 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Balanced + - 8192 / 1024 + - 128 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 8192 / 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` + * - nvidia/DeepSeek-R1-0528-FP4-v2 + - B200_NVL + - Max Throughput + - 8192 / 1024 + - 256 + - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - B200_NVL + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 1024 / 8192 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 1024 / 8192 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 1024 / 8192 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 4 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 8 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Min Latency + - 8192 / 1024 + - 16 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Balanced + - 8192 / 1024 + - 32 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` + * - openai/gpt-oss-120b + - H200_SXM + - Max Throughput + - 8192 / 1024 + - 64 + - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` From c1b20d01a3064a6dec322a9e90ccc940dd3605c7 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 12:56:09 -0800 Subject: [PATCH 11/39] fix pass Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide/comprehensive_table.rst | 543 +++++++----------- tools/generate_config_table.py | 157 ++--- 2 files changed, 290 insertions(+), 410 deletions(-) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 875b92bb85a6..432819eb8365 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -1,1185 +1,1050 @@ +deepseek-ai/DeepSeek-R1-0528 +============================ + .. list-table:: + :width: 100% :header-rows: 1 - :widths: 20 12 12 15 10 20 25 + :widths: 12 15 18 10 20 25 - * - Model Name - - GPU + * - GPU - Performance Profile - ISL / OSL - Concurrency - Config - Command - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL + * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM + * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + +nvidia/DeepSeek-R1-0528-FP4-v2 +============================== + +.. list-table:: + :width: 100% + :header-rows: 1 + :widths: 12 15 18 10 20 25 + + * - GPU + - Performance Profile + - ISL / OSL + - Concurrency + - Config + - Command + * - B200_NVL - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 32 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 32 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 64 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 64 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Balanced - 1024 / 1024 - 128 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Balanced - 1024 / 1024 - 128 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 1024 - 256 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 1024 - 256 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 32 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 32 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 64 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 64 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Balanced - 8192 / 1024 - 128 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Balanced - 8192 / 1024 - 128 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Max Throughput - 8192 / 1024 - 256 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL + * - B200_NVL - Max Throughput - 8192 / 1024 - 256 - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + +openai/gpt-oss-120b +=================== + +.. list-table:: + :width: 100% + :header-rows: 1 + :widths: 12 15 18 10 20 25 + + * - GPU + - Performance Profile + - ISL / OSL + - Concurrency + - Config + - Command + * - B200_NVL - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL + * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 1024 / 8192 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 1024 / 8192 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 4 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 8 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Min Latency - 8192 / 1024 - 16 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Balanced - 8192 / 1024 - 32 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM + * - H200_SXM - Max Throughput - 8192 / 1024 - 64 diff --git a/tools/generate_config_table.py b/tools/generate_config_table.py index 3e1947ba4a5c..171a74721771 100644 --- a/tools/generate_config_table.py +++ b/tools/generate_config_table.py @@ -15,82 +15,97 @@ def generate_rst(yaml_path, output_file=None): with open(yaml_path, "r") as f: data = yaml.safe_load(f) - # Group by key attributes to determine min/max concurrency - # Key: (model, gpu, isl, osl) -> list of entry - groups = defaultdict(list) + # Group by model first, then by key attributes + # Structure: model -> (gpu, isl, osl) -> list of entries + model_groups = defaultdict(lambda: defaultdict(list)) for entry in data: - key = (entry.get("model"), entry.get("gpu"), entry.get("isl"), entry.get("osl")) - groups[key].append(entry) + model = entry.get("model", "Unknown Model") + key = (entry.get("gpu"), entry.get("isl"), entry.get("osl")) + model_groups[model][key].append(entry) # Prepare output lines lines = [] - lines.append(".. list-table::") - lines.append(" :header-rows: 1") - lines.append(" :widths: 20 12 12 15 10 20 25") - lines.append("") - lines.append(" * - Model Name") - lines.append(" - GPU") - lines.append(" - Performance Profile") - lines.append(" - ISL / OSL") - lines.append(" - Concurrency") - lines.append(" - Config") - lines.append(" - Command") - - # Sort keys for consistent output - # Key elements might be None, so convert to str/int safely - sorted_keys = sorted( - groups.keys(), key=lambda k: (str(k[0]), str(k[1]), int(k[2] or 0), int(k[3] or 0)) - ) - - for key in sorted_keys: - entries = groups[key] - # Sort by concurrency - entries.sort(key=lambda x: int(x.get("concurrency", 0))) - - # Get concurrency range for this group - min_conc = int(entries[0].get("concurrency", 0)) - max_conc = int(entries[-1].get("concurrency", 0)) - conc_range = max_conc - min_conc - - for entry in entries: - model = entry.get("model", "N/A") - gpu = entry.get("gpu", "N/A") - isl = entry.get("isl", "N/A") - osl = entry.get("osl", "N/A") - conc = int(entry.get("concurrency", 0)) - config_path = entry.get("config_path", "") - - # Determine profile based on relative position in concurrency range - if len(entries) == 1: - # Single entry: use concurrency value as heuristic - # Lower concurrency typically means lower latency - if conc <= 16: - profile = "Min Latency" - elif conc >= 64: - profile = "Max Throughput" - else: - profile = "Balanced" - else: - # Multiple entries: use relative position - relative_pos = (conc - min_conc) / conc_range if conc_range > 0 else 0.5 - - if relative_pos < 0.33: - profile = "Min Latency" - elif relative_pos > 0.67: - profile = "Max Throughput" + + # Sort models alphabetically + sorted_models = sorted(model_groups.keys()) + + for model in sorted_models: + # Section Header for Model + lines.append(model) + lines.append("=" * len(model)) + lines.append("") + + # Table Header + lines.append(".. list-table::") + lines.append(" :width: 100%") + lines.append(" :header-rows: 1") + # Adjusted widths removing 'Model Name' (originally 20) + # Distributing remaining space roughly to Config/Command which are long + lines.append(" :widths: 12 15 18 10 20 25") + lines.append("") + lines.append(" * - GPU") + lines.append(" - Performance Profile") + lines.append(" - ISL / OSL") + lines.append(" - Concurrency") + lines.append(" - Config") + lines.append(" - Command") + + # Process entries for this model + subgroups = model_groups[model] + + # Sort subgroups by GPU, ISL, OSL + sorted_keys = sorted( + subgroups.keys(), key=lambda k: (str(k[0]), int(k[1] or 0), int(k[2] or 0)) + ) + + for key in sorted_keys: + entries = subgroups[key] + # Sort by concurrency + entries.sort(key=lambda x: int(x.get("concurrency", 0))) + + # Get concurrency range for this group to determine profile + min_conc = int(entries[0].get("concurrency", 0)) + max_conc = int(entries[-1].get("concurrency", 0)) + conc_range = max_conc - min_conc + + for entry in entries: + gpu = entry.get("gpu", "N/A") + isl = entry.get("isl", "N/A") + osl = entry.get("osl", "N/A") + conc = int(entry.get("concurrency", 0)) + config_path = entry.get("config_path", "") + + # Determine profile based on relative position in concurrency range + if len(entries) == 1: + # Single entry: use concurrency value as heuristic + if conc <= 16: + profile = "Min Latency" + elif conc >= 64: + profile = "Max Throughput" + else: + profile = "Balanced" else: - profile = "Balanced" - - full_config_path = os.path.join("tensorrt_llm/configure", config_path) - command = f"trtllm-serve {model} --extra_llm_api_options {full_config_path}" - - lines.append(f" * - {model}") - lines.append(f" - {gpu}") - lines.append(f" - {profile}") - lines.append(f" - {isl} / {osl}") - lines.append(f" - {conc}") - lines.append(f" - {full_config_path}") - lines.append(f" - ``{command}``") + # Multiple entries: use relative position + relative_pos = (conc - min_conc) / conc_range if conc_range > 0 else 0.5 + + if relative_pos < 0.33: + profile = "Min Latency" + elif relative_pos > 0.67: + profile = "Max Throughput" + else: + profile = "Balanced" + + full_config_path = os.path.join("tensorrt_llm/configure", config_path) + command = f"trtllm-serve {model} --extra_llm_api_options {full_config_path}" + + lines.append(f" * - {gpu}") + lines.append(f" - {profile}") + lines.append(f" - {isl} / {osl}") + lines.append(f" - {conc}") + lines.append(f" - {full_config_path}") + lines.append(f" - ``{command}``") + + lines.append("") # Space between tables # Output to file or stdout output_text = "\n".join(lines) From 688b07cc32ba11105897bbe245f2c1f4793dc9b6 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 16:23:32 -0800 Subject: [PATCH 12/39] enhancememnts Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide/comprehensive_table.rst | 360 +++++++++--------- ...loyment-guide-for-deepseek-r1-on-trtllm.md | 16 + .../deployment-guide-for-gpt-oss-on-trtllm.md | 10 + tools/generate_config_table.py | 14 +- 4 files changed, 227 insertions(+), 173 deletions(-) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 432819eb8365..fcc5147d60f1 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -1,5 +1,9 @@ +.. start-deepseek-ai/DeepSeek-R1-0528 + +.. _deepseek-ai/DeepSeek-R1-0528: + deepseek-ai/DeepSeek-R1-0528 -============================ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :width: 100% @@ -16,125 +20,131 @@ deepseek-ai/DeepSeek-R1-0528 - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml + - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml + - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml + - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml + - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml + - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml + - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml + - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml + - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml + - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml + - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml + - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml + - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml + - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml + - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml + - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml + - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml + - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml + - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml + - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml + - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` +.. end-deepseek-ai/DeepSeek-R1-0528 + +.. start-nvidia/DeepSeek-R1-0528-FP4-v2 + +.. _nvidia/DeepSeek-R1-0528-FP4-v2: + nvidia/DeepSeek-R1-0528-FP4-v2 -============================== +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :width: 100% @@ -151,173 +161,179 @@ nvidia/DeepSeek-R1-0528-FP4-v2 - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml + - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` +.. end-nvidia/DeepSeek-R1-0528-FP4-v2 + +.. start-openai/gpt-oss-120b + +.. _openai/gpt-oss-120b: + openai/gpt-oss-120b -=================== +^^^^^^^^^^^^^^^^^^^ .. list-table:: :width: 100% @@ -334,719 +350,721 @@ openai/gpt-oss-120b - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` * - B200_NVL - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` * - B200_NVL - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` * - B200_NVL - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml + - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` * - H200_SXM - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` * - H200_SXM - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` * - H200_SXM - Balanced - 1024 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` * - H200_SXM - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` * - H200_SXM - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` * - H200_SXM - Balanced - 1024 / 8192 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` * - H200_SXM - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` * - H200_SXM - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` * - H200_SXM - Balanced - 8192 / 1024 - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml + - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` + +.. end-openai/gpt-oss-120b diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index 3a25c5c7526e..3a9edd25759b 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -110,6 +110,22 @@ append: EOF ``` ```` +### Pre-configured Scenarios + +The following tables list recommended configurations from the comprehensive database for different performance profiles. + +```{eval-rst} +.. include:: comprehensive_table.rst + :start-after: .. start-deepseek-ai/DeepSeek-R1-0528 + :end-before: .. end-deepseek-ai/DeepSeek-R1-0528 +``` + +```{eval-rst} +.. include:: comprehensive_table.rst + :start-after: .. start-nvidia/DeepSeek-R1-0528-FP4-v2 + :end-before: .. end-nvidia/DeepSeek-R1-0528-FP4-v2 +``` + ### Launch the TensorRT LLM Server Below is an example command to launch the TensorRT LLM server with the DeepSeek-R1 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “LLM API Options (YAML Configuration)” section. diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index 86fc4bc786ad..e3a0b1e4dbf2 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -108,6 +108,16 @@ append: EOF ``` ```` +### Pre-configured Scenarios + +The following table lists recommended configurations from the comprehensive database for different performance profiles. + +```{eval-rst} +.. include:: comprehensive_table.rst + :start-after: .. start-openai/gpt-oss-120b + :end-before: .. end-openai/gpt-oss-120b +``` + ### Launch the TensorRT LLM Server Below is an example command to launch the TensorRT LLM server with the GPT-OSS model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “LLM API Options (YAML Configuration)” section. diff --git a/tools/generate_config_table.py b/tools/generate_config_table.py index 171a74721771..07cf9524f2bc 100644 --- a/tools/generate_config_table.py +++ b/tools/generate_config_table.py @@ -30,9 +30,13 @@ def generate_rst(yaml_path, output_file=None): sorted_models = sorted(model_groups.keys()) for model in sorted_models: + lines.append(f".. start-{model}") + lines.append("") # Section Header for Model + lines.append(f".. _{model}:") + lines.append("") lines.append(model) - lines.append("=" * len(model)) + lines.append("^" * len(model)) lines.append("") # Table Header @@ -98,14 +102,20 @@ def generate_rst(yaml_path, output_file=None): full_config_path = os.path.join("tensorrt_llm/configure", config_path) command = f"trtllm-serve {model} --extra_llm_api_options {full_config_path}" + config_filename = os.path.basename(full_config_path) + github_url = f"https://github.com/NVIDIA/TensorRT-LLM/blob/main/{full_config_path}" + config_link = f"`{config_filename} <{github_url}>`_" + lines.append(f" * - {gpu}") lines.append(f" - {profile}") lines.append(f" - {isl} / {osl}") lines.append(f" - {conc}") - lines.append(f" - {full_config_path}") + lines.append(f" - {config_link}") lines.append(f" - ``{command}``") lines.append("") # Space between tables + lines.append(f".. end-{model}") + lines.append("") # Output to file or stdout output_text = "\n".join(lines) From 921d72acbfa75a9dea20024fbf0e202b5f16fb19 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 17:19:32 -0800 Subject: [PATCH 13/39] more enhancements Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide/comprehensive_table.rst | 702 +++++++++--------- ...loyment-guide-for-deepseek-r1-on-trtllm.md | 34 +- .../deployment-guide-for-gpt-oss-on-trtllm.md | 20 +- {tools => scripts}/generate_config_table.py | 51 +- 4 files changed, 420 insertions(+), 387 deletions(-) rename {tools => scripts}/generate_config_table.py (77%) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index fcc5147d60f1..6cf497fbbb22 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -2,140 +2,140 @@ .. _deepseek-ai/DeepSeek-R1-0528: -deepseek-ai/DeepSeek-R1-0528 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +`DeepSeek-R1 `_ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :width: 100% :header-rows: 1 - :widths: 12 15 18 10 20 25 + :widths: 12 15 20 25 15 13 * - GPU - Performance Profile - - ISL / OSL - - Concurrency - Config - Command + - Best ISL / OSL + - Best Concurrency * - B200_NVL - Min Latency + - `fp8_b200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 - - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency + - `fp8_b200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 - - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency + - `fp8_b200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 - - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced + - `fp8_b200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 - - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput + - `fp8_b200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 - - `deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Min Latency + - `fp8_b200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 - - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency + - `fp8_b200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 - - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency + - `fp8_b200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 - - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced + - `fp8_b200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 - - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput + - `fp8_b200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 - - `deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency + - `fp8_h200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 - - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency + - `fp8_h200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 - - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency + - `fp8_h200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 - - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced + - `fp8_h200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 - - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput + - `fp8_h200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 - - `deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency + - `fp8_h200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 - - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency + - `fp8_h200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 - - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency + - `fp8_h200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 - - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced + - `fp8_h200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 - - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput + - `fp8_h200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 - - `deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` .. end-deepseek-ai/DeepSeek-R1-0528 @@ -143,188 +143,188 @@ deepseek-ai/DeepSeek-R1-0528 .. _nvidia/DeepSeek-R1-0528-FP4-v2: -nvidia/DeepSeek-R1-0528-FP4-v2 -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +`DeepSeek-R1 (NVFP4) `_ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :width: 100% :header-rows: 1 - :widths: 12 15 18 10 20 25 + :widths: 12 15 20 25 15 13 * - GPU - Performance Profile - - ISL / OSL - - Concurrency - Config - Command + - Best ISL / OSL + - Best Concurrency * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` - 1024 / 1024 - 128 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` - 1024 / 1024 - 128 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` - 1024 / 1024 - 256 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` - 1024 / 1024 - 256 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_8k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` - 8192 / 1024 - 128 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_8k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` - 8192 / 1024 - 128 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_8k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` - 8192 / 1024 - 256 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_8k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` - 8192 / 1024 - 256 - - `deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` .. end-nvidia/DeepSeek-R1-0528-FP4-v2 @@ -332,739 +332,739 @@ nvidia/DeepSeek-R1-0528-FP4-v2 .. _openai/gpt-oss-120b: -openai/gpt-oss-120b -^^^^^^^^^^^^^^^^^^^ +`gpt-oss-120b `_ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :width: 100% :header-rows: 1 - :widths: 12 15 18 10 20 25 + :widths: 12 15 20 25 15 13 * - GPU - Performance Profile - - ISL / OSL - - Concurrency - Config - Command + - Best ISL / OSL + - Best Concurrency * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency + - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` * - B200_NVL - Balanced + - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput + - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` - 1024 / 8192 - 4 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` - 1024 / 8192 - 8 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` - 1024 / 8192 - 16 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` - 1024 / 8192 - 32 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` - 1024 / 8192 - 64 - - `gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` * - H200_SXM - Min Latency + - `fp4_h200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` * - H200_SXM - Balanced + - `fp4_h200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput + - `fp4_h200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 - - `gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` .. end-openai/gpt-oss-120b diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index 3a9edd25759b..8386a7c98d1e 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -110,22 +110,6 @@ append: EOF ``` ```` -### Pre-configured Scenarios - -The following tables list recommended configurations from the comprehensive database for different performance profiles. - -```{eval-rst} -.. include:: comprehensive_table.rst - :start-after: .. start-deepseek-ai/DeepSeek-R1-0528 - :end-before: .. end-deepseek-ai/DeepSeek-R1-0528 -``` - -```{eval-rst} -.. include:: comprehensive_table.rst - :start-after: .. start-nvidia/DeepSeek-R1-0528-FP4-v2 - :end-before: .. end-nvidia/DeepSeek-R1-0528-FP4-v2 -``` - ### Launch the TensorRT LLM Server Below is an example command to launch the TensorRT LLM server with the DeepSeek-R1 model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “LLM API Options (YAML Configuration)” section. @@ -170,7 +154,7 @@ These options provide control over TensorRT LLM's behavior and are set within th #### `trust_remote_code` - **Description:** Allows TensorRT LLM to download models and tokenizers from Hugging Face. This flag is passed directly to the Hugging Face API. +* **Description:** Allows TensorRT LLM to download models and tokenizers from Hugging Face. This flag is passed directly to the Hugging Face API. #### `kv_cache_config` @@ -445,3 +429,19 @@ $$ $$ \text{TPS} = \frac{\text{Num Output Tokens}}{T_{last} - T_{first}} $$ + +## Preconfigured Recipes + +The following tables list recommended configurations from the comprehensive database for different performance profiles. + +```{eval-rst} +.. include:: comprehensive_table.rst + :start-after: .. start-deepseek-ai/DeepSeek-R1-0528 + :end-before: .. end-deepseek-ai/DeepSeek-R1-0528 +``` + +```{eval-rst} +.. include:: comprehensive_table.rst + :start-after: .. start-nvidia/DeepSeek-R1-0528-FP4-v2 + :end-before: .. end-nvidia/DeepSeek-R1-0528-FP4-v2 +``` diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index e3a0b1e4dbf2..34e097f350cf 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -108,16 +108,6 @@ append: EOF ``` ```` -### Pre-configured Scenarios - -The following table lists recommended configurations from the comprehensive database for different performance profiles. - -```{eval-rst} -.. include:: comprehensive_table.rst - :start-after: .. start-openai/gpt-oss-120b - :end-before: .. end-openai/gpt-oss-120b -``` - ### Launch the TensorRT LLM Server Below is an example command to launch the TensorRT LLM server with the GPT-OSS model from within the container. The command is specifically configured for the 1024/1024 Input/Output Sequence Length test. The explanation of each flag is shown in the “LLM API Options (YAML Configuration)” section. @@ -387,3 +377,13 @@ $$ $$ \text{TPS} = \frac{\text{Num Output Tokens}}{T_{last} - T_{first}} $$ + +## Preconfigured Recipes + +The following table lists recommended configurations from the comprehensive database for different performance profiles. + +```{eval-rst} +.. include:: comprehensive_table.rst + :start-after: .. start-openai/gpt-oss-120b + :end-before: .. end-openai/gpt-oss-120b +``` diff --git a/tools/generate_config_table.py b/scripts/generate_config_table.py similarity index 77% rename from tools/generate_config_table.py rename to scripts/generate_config_table.py index 07cf9524f2bc..2090dde7b857 100644 --- a/tools/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -4,6 +4,25 @@ import yaml +# Mapping for model display names, URLs, and filename shortening prefixes +MODEL_INFO = { + "deepseek-ai/DeepSeek-R1-0528": { + "display_name": "DeepSeek-R1", + "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528", + "prefix_to_strip": "deepseek_r1_0528_", + }, + "nvidia/DeepSeek-R1-0528-FP4-v2": { + "display_name": "DeepSeek-R1 (NVFP4)", + "url": "https://huggingface.co/nvidia/DeepSeek-R1-FP4-v2", + "prefix_to_strip": "deepseek_r1_0528_fp4_v2_", + }, + "openai/gpt-oss-120b": { + "display_name": "gpt-oss-120b", + "url": "https://huggingface.co/openai/gpt-oss-120b", + "prefix_to_strip": "gpt_oss_120b_", + }, +} + def generate_rst(yaml_path, output_file=None): """Generate RST table from YAML config database. @@ -32,27 +51,34 @@ def generate_rst(yaml_path, output_file=None): for model in sorted_models: lines.append(f".. start-{model}") lines.append("") + + # Determine title text + if model in MODEL_INFO: + info = MODEL_INFO[model] + title_text = f"`{info['display_name']} <{info['url']}>`_" + else: + title_text = model + # Section Header for Model lines.append(f".. _{model}:") lines.append("") - lines.append(model) - lines.append("^" * len(model)) + lines.append(title_text) + lines.append("^" * len(title_text)) lines.append("") # Table Header lines.append(".. list-table::") lines.append(" :width: 100%") lines.append(" :header-rows: 1") - # Adjusted widths removing 'Model Name' (originally 20) - # Distributing remaining space roughly to Config/Command which are long - lines.append(" :widths: 12 15 18 10 20 25") + # Widths: GPU, Perf Profile, Config, Command, Recommended Request Settings (ISL/OSL, Concurrency) + lines.append(" :widths: 12 15 20 25 15 13") lines.append("") lines.append(" * - GPU") lines.append(" - Performance Profile") - lines.append(" - ISL / OSL") - lines.append(" - Concurrency") lines.append(" - Config") lines.append(" - Command") + lines.append(" - Best ISL / OSL") + lines.append(" - Best Concurrency") # Process entries for this model subgroups = model_groups[model] @@ -103,15 +129,22 @@ def generate_rst(yaml_path, output_file=None): command = f"trtllm-serve {model} --extra_llm_api_options {full_config_path}" config_filename = os.path.basename(full_config_path) + + # Shorten config filename if applicable + if model in MODEL_INFO: + prefix = MODEL_INFO[model].get("prefix_to_strip", "") + if config_filename.startswith(prefix): + config_filename = config_filename[len(prefix) :] + github_url = f"https://github.com/NVIDIA/TensorRT-LLM/blob/main/{full_config_path}" config_link = f"`{config_filename} <{github_url}>`_" lines.append(f" * - {gpu}") lines.append(f" - {profile}") - lines.append(f" - {isl} / {osl}") - lines.append(f" - {conc}") lines.append(f" - {config_link}") lines.append(f" - ``{command}``") + lines.append(f" - {isl} / {osl}") + lines.append(f" - {conc}") lines.append("") # Space between tables lines.append(f".. end-{model}") From 3565377e700739a180bcbd98cc47795cf8a01b8a Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 18:10:11 -0800 Subject: [PATCH 14/39] more enhancements Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- comprehensive_table.rst | 1356 ----------------- .../deployment-guide/comprehensive_table.rst | 336 ++-- docs/source/deployment-guide/index.rst | 7 +- scripts/generate_config_table.py | 4 +- 4 files changed, 177 insertions(+), 1526 deletions(-) delete mode 100644 comprehensive_table.rst diff --git a/comprehensive_table.rst b/comprehensive_table.rst deleted file mode 100644 index f4c66e0a0963..000000000000 --- a/comprehensive_table.rst +++ /dev/null @@ -1,1356 +0,0 @@ -.. list-table:: - :header-rows: 1 - :widths: 15 10 12 6 6 6 20 25 - - * - Model Name - - GPU - - Performance Profile - - ISL - - OSL - - Concurrency - - Config - - Command - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Max Throughput - - 1024 - - 1024 - - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Max Throughput - - 1024 - - 1024 - - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Max Throughput - - 8192 - - 1024 - - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 128 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Max Throughput - - 8192 - - 1024 - - 256 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - * - nvidia/DeepSeek-R1-0528-FP4-v2 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - B200_NVL - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` - * - deepseek-ai/DeepSeek-R1-0528 - - H200_SXM - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - B200_NVL - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 1024 - - 8192 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 1024 - - 8192 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 1024 - - 8192 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 16 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 32 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Min Latency - - 8192 - - 1024 - - 4 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Max Throughput - - 8192 - - 1024 - - 64 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` - * - openai/gpt-oss-120b - - H200_SXM - - Balanced - - 8192 - - 1024 - - 8 - - tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 6cf497fbbb22..cd3c310e8b17 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -19,121 +19,121 @@ * - B200_NVL - Min Latency - `fp8_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 * - B200_NVL - Min Latency - `fp8_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 * - B200_NVL - Min Latency - `fp8_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 * - B200_NVL - Balanced - `fp8_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 * - B200_NVL - Max Throughput - `fp8_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 * - B200_NVL - Min Latency - `fp8_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 * - B200_NVL - Min Latency - `fp8_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 * - B200_NVL - Min Latency - `fp8_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 * - B200_NVL - Balanced - `fp8_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 * - B200_NVL - Max Throughput - `fp8_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 * - H200_SXM - Min Latency - `fp8_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 * - H200_SXM - Min Latency - `fp8_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 * - H200_SXM - Min Latency - `fp8_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 * - H200_SXM - Balanced - `fp8_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 * - H200_SXM - Max Throughput - `fp8_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 * - H200_SXM - Min Latency - `fp8_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 * - H200_SXM - Min Latency - `fp8_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 * - H200_SXM - Min Latency - `fp8_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 * - H200_SXM - Balanced - `fp8_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 * - H200_SXM - Max Throughput - `fp8_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 @@ -160,169 +160,169 @@ * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 * - B200_NVL - Balanced - `fp4_b200_trt_1k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` - 1024 / 1024 - 128 * - B200_NVL - Balanced - `fp4_b200_trt_1k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` - 1024 / 1024 - 128 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` - 1024 / 1024 - 256 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` - 1024 / 1024 - 256 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 * - B200_NVL - Balanced - `fp4_b200_trt_8k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` - 8192 / 1024 - 128 * - B200_NVL - Balanced - `fp4_b200_trt_8k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` - 8192 / 1024 - 128 * - B200_NVL - Max Throughput - `fp4_b200_trt_8k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` - 8192 / 1024 - 256 * - B200_NVL - Max Throughput - `fp4_b200_trt_8k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` - 8192 / 1024 - 256 @@ -349,721 +349,721 @@ * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` - 1024 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` - 1024 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` - 1024 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` - 1024 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` - 1024 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` - 1024 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 * - B200_NVL - Balanced - `fp4_b200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` - 1024 / 1024 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` - 1024 / 1024 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` - 1024 / 1024 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` - 1024 / 1024 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` - 1024 / 8192 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` - 1024 / 8192 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` - 1024 / 8192 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` - 1024 / 8192 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` - 1024 / 8192 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` - 1024 / 8192 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` - 1024 / 8192 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` - 1024 / 8192 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` - 1024 / 8192 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` - 1024 / 8192 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` - 1024 / 8192 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` - 1024 / 8192 - 16 * - B200_NVL - Balanced - `fp4_b200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` - 1024 / 8192 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` - 1024 / 8192 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` - 1024 / 8192 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` - 1024 / 8192 - 32 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` - 1024 / 8192 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` - 1024 / 8192 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` - 1024 / 8192 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` - 1024 / 8192 - 64 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` - 8192 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` - 8192 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` - 8192 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` - 8192 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` - 8192 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` - 8192 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 * - B200_NVL - Min Latency - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 * - B200_NVL - Balanced - `fp4_b200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` - 8192 / 1024 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` - 8192 / 1024 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 * - B200_NVL - Balanced - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 * - B200_NVL - Max Throughput - `fp4_b200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` - 8192 / 1024 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` - 8192 / 1024 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 * - B200_NVL - Max Throughput - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` - 1024 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` - 1024 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` - 1024 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` - 1024 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` - 1024 / 1024 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` - 1024 / 1024 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 * - H200_SXM - Balanced - `fp4_h200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` - 1024 / 1024 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` - 1024 / 1024 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` - 1024 / 1024 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` - 1024 / 1024 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` - 1024 / 8192 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` - 1024 / 8192 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` - 1024 / 8192 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` - 1024 / 8192 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` - 1024 / 8192 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` - 1024 / 8192 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` - 1024 / 8192 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` - 1024 / 8192 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` - 1024 / 8192 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` - 1024 / 8192 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` - 1024 / 8192 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` - 1024 / 8192 - 16 * - H200_SXM - Balanced - `fp4_h200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` - 1024 / 8192 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` - 1024 / 8192 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` - 1024 / 8192 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` - 1024 / 8192 - 32 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` - 1024 / 8192 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` - 1024 / 8192 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` - 1024 / 8192 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` - 1024 / 8192 - 64 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` - 8192 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` - 8192 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` - 8192 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` - 8192 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` - 8192 / 1024 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` - 8192 / 1024 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 * - H200_SXM - Min Latency - `fp4_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 * - H200_SXM - Balanced - `fp4_h200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` - 8192 / 1024 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` - 8192 / 1024 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 * - H200_SXM - Balanced - `fp4_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 * - H200_SXM - Max Throughput - `fp4_h200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` - 8192 / 1024 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` - 8192 / 1024 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 * - H200_SXM - Max Throughput - `fp4_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 diff --git a/docs/source/deployment-guide/index.rst b/docs/source/deployment-guide/index.rst index e43f83612a78..60a224fbc418 100644 --- a/docs/source/deployment-guide/index.rst +++ b/docs/source/deployment-guide/index.rst @@ -6,7 +6,12 @@ Quick Start for Popular Models The table below contains ``trtllm-serve`` commands that can be used to easily deploy popular models including DeepSeek-R1, gpt-oss, Llama 4, Qwen3, and more. -We maintain LLM API configuration files for these models containing recommended performance settings in the `examples/configs `_ directory. The TensorRT LLM Docker container makes the config files available at ``/app/tensorrt_llm/examples/configs``, but you can customize this as needed: +We maintain LLM API configuration files for these models containing recommended performance settings in two locations: + +* **Curated Examples**: `examples/configs `_ - Hand-picked configurations for common scenarios. +* **Comprehensive Database**: `tensorrt_llm/configure/database `_ - A more comprehensive set of known-good configurations for various GPUs and inference scenarios. + +The TensorRT LLM Docker container makes these config files available at ``/app/tensorrt_llm/examples/configs`` and ``/app/tensorrt_llm/tensorrt_llm/configure/database`` respectively. You can reference them as needed: .. code-block:: bash diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 2090dde7b857..15113f823308 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -126,7 +126,9 @@ def generate_rst(yaml_path, output_file=None): profile = "Balanced" full_config_path = os.path.join("tensorrt_llm/configure", config_path) - command = f"trtllm-serve {model} --extra_llm_api_options {full_config_path}" + command = ( + f"trtllm-serve {model} --extra_llm_api_options ${{TRTLLM_DIR}}/{config_path}" + ) config_filename = os.path.basename(full_config_path) From c400927d484b67b7a0162170ff0bd8342919c973 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 21:45:55 -0800 Subject: [PATCH 15/39] add unittest Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tests/unittest/test_generate_config_table.py | 52 ++++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 tests/unittest/test_generate_config_table.py diff --git a/tests/unittest/test_generate_config_table.py b/tests/unittest/test_generate_config_table.py new file mode 100644 index 000000000000..a3ef0fca6607 --- /dev/null +++ b/tests/unittest/test_generate_config_table.py @@ -0,0 +1,52 @@ +import os +import sys +import tempfile +import unittest + +# Add scripts directory to path +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +SCRIPTS_DIR = os.path.join(REPO_ROOT, "scripts") +sys.path.insert(0, SCRIPTS_DIR) + +# Try to import generate_rst, fail gracefully if dependencies (like yaml) are missing +from generate_config_table import generate_rst # noqa: E402 + + +class TestConfigTableSync(unittest.TestCase): + def test_config_table_sync(self): + """Test that the comprehensive_table.rst file is synchronized with the scenario_list.yaml database. + + Ensures that the RST file is up-to-date with the YAML database. + """ + if generate_rst is None: + self.skipTest("generate_config_table not available") + + # Define paths + yaml_path = os.path.join(REPO_ROOT, "tensorrt_llm/configure/database/scenario_list.yaml") + rst_path = os.path.join(REPO_ROOT, "docs/source/deployment-guide/comprehensive_table.rst") + + # Ensure files exist + self.assertTrue(os.path.exists(yaml_path), f"YAML file not found: {yaml_path}") + self.assertTrue(os.path.exists(rst_path), f"RST file not found: {rst_path}") + + # Read existing RST content + with open(rst_path, "r") as f: + existing_content = f.read() + + # Generate new RST content + with tempfile.NamedTemporaryFile(mode="w+", delete=True) as tmp: + generate_rst(yaml_path, output_file=tmp.name) + tmp.seek(0) + generated_content = tmp.read() + + # Compare content + self.assertEqual( + existing_content.strip(), + generated_content.strip(), + "comprehensive_table.rst is not synchronized with scenario_list.yaml. " + "Please run 'python3 scripts/generate_config_table.py' from the repo root to update it.", + ) + + +if __name__ == "__main__": + unittest.main() From ebcd251cdd56b9483a3a9404bca94b97b9ca0fcb Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 21:48:53 -0800 Subject: [PATCH 16/39] include missing config options Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 6 ++++++ .../B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 6 ++++++ .../H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 6 ++++++ ...epseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 6 ++++++ ...eepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 6 ++++++ ...deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 6 ++++++ .../B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 6 ++++++ .../H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 6 ++++++ 168 files changed, 1008 insertions(+) diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml index cea36102a893..f770a6566e93 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml index cea36102a893..f770a6566e93 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml index cea36102a893..f770a6566e93 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml index cea36102a893..f770a6566e93 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml index cea36102a893..f770a6566e93 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml index cea36102a893..6660bcea96ae 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml index cea36102a893..6660bcea96ae 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml index cea36102a893..6660bcea96ae 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml index 87ec25cb384b..919a0284099e 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml index cea36102a893..6660bcea96ae 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml index c10047cbd6be..008da1df54c4 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml index c10047cbd6be..008da1df54c4 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml index c10047cbd6be..008da1df54c4 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml index c10047cbd6be..008da1df54c4 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml index c10047cbd6be..008da1df54c4 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml index c10047cbd6be..decbb1744a3a 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml index c10047cbd6be..decbb1744a3a 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml index c10047cbd6be..decbb1744a3a 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml index 267ee4500f9c..363eebf52142 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml index c10047cbd6be..decbb1744a3a 100644 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml index fa392eff1197..c61e3abc15ed 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1216 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml index fa392eff1197..fe58a6a32b27 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml index 33f5153ce191..2a06d3978d01 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1344 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml index fa392eff1197..fe58a6a32b27 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml index fa392eff1197..fe58a6a32b27 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml index fa392eff1197..fe58a6a32b27 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml index fa392eff1197..fe58a6a32b27 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml index fa392eff1197..a4a4fe28c7fb 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1216 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml index fa392eff1197..397565e15b50 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml index 33f5153ce191..686db04f1fbd 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1344 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml index fa392eff1197..397565e15b50 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml index fa392eff1197..397565e15b50 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml index fa392eff1197..397565e15b50 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml index fa392eff1197..397565e15b50 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml index 33f5153ce191..ace419c0d81e 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8384 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml index fa392eff1197..a0f2de5fec79 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml index 33f5153ce191..3c812ea3e963 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8512 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml index fa392eff1197..a0f2de5fec79 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml index fa392eff1197..a0f2de5fec79 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml index 33f5153ce191..06f600c1cd90 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml index fa392eff1197..a0f2de5fec79 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml index 33f5153ce191..5334ed3cf545 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8384 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml index fa392eff1197..382a3c9045da 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml index 33f5153ce191..639fdde94a18 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8512 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml index fa392eff1197..382a3c9045da 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml index fa392eff1197..382a3c9045da 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml index 33f5153ce191..930a6253084f 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -14,3 +14,9 @@ attention_dp_config: batching_wait_iters: 0 enable_balance: true timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml index fa392eff1197..382a3c9045da 100644 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -10,3 +10,9 @@ kv_cache_config: stream_interval: 10 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml index 2a5db8ffba68..1d4df970107d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml index 93aa390c8825..7d65f54710bf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml index fa26d3b12649..ca850a775852 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml index 46baa7be7143..345b0e501388 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml index 59cb144ec424..5fa5e373d2e5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml index 2a5db8ffba68..7b392ada8d6b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml index 93aa390c8825..e8212dd139ff 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml index fa26d3b12649..ab22a7baf682 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml index 46baa7be7143..3f8265048033 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml index 59cb144ec424..b07960f33ddd 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml index 2a5db8ffba68..e078ea3d6d32 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml index 93aa390c8825..15f5a3ca501d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml index fa26d3b12649..cdbb40a3eb47 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml index 46baa7be7143..c5854b6dafb1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml index 59cb144ec424..0ac44311750b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml index 2a5db8ffba68..a18faa262246 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml index 93aa390c8825..4ce42b3ce83f 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml index fa26d3b12649..966138c163c4 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml index 46baa7be7143..a322f0681d28 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml index 59cb144ec424..644d2dabb489 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml index 2a5db8ffba68..31544aa9f4f6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml index 93aa390c8825..ec0ea7b2ba31 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml index fa26d3b12649..249b14723ff3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml index 46baa7be7143..21de3414a842 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml index 59cb144ec424..315b1add429c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml index 2a5db8ffba68..56e1b648bd78 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml index 93aa390c8825..4e02fe671b4e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml index fa26d3b12649..4bc360839a97 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml index 46baa7be7143..584fb5ae1aee 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml index 59cb144ec424..6ab46126d56b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml index 2a5db8ffba68..ef539d3befa9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml index 93aa390c8825..40dc75208448 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml index fa26d3b12649..3e0f48e7e1bf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml index 46baa7be7143..2e3721c712dc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml index 59cb144ec424..098e7ec388cf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml index 2a5db8ffba68..45d77f70bd23 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml index 93aa390c8825..9436b079590a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml index fa26d3b12649..a2917bfd5b19 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml index 46baa7be7143..702d3bc00cb1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml index 59cb144ec424..c0b90314c3d0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml index 2a5db8ffba68..31544aa9f4f6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml index 93aa390c8825..ec0ea7b2ba31 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml index fa26d3b12649..249b14723ff3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml index 46baa7be7143..21de3414a842 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml index 59cb144ec424..315b1add429c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml index 2a5db8ffba68..56e1b648bd78 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml index 93aa390c8825..4e02fe671b4e 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml index fa26d3b12649..4bc360839a97 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml index 46baa7be7143..584fb5ae1aee 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml index 59cb144ec424..6ab46126d56b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml index 2a5db8ffba68..ef539d3befa9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml index 93aa390c8825..40dc75208448 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml index fa26d3b12649..3e0f48e7e1bf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml index 46baa7be7143..2e3721c712dc 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml index 59cb144ec424..098e7ec388cf 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml index 2a5db8ffba68..45d77f70bd23 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml index 93aa390c8825..9436b079590a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml index fa26d3b12649..a2917bfd5b19 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml index 46baa7be7143..702d3bc00cb1 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml index 59cb144ec424..c0b90314c3d0 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml @@ -14,3 +14,9 @@ stream_interval: 20 num_postprocess_workers: 4 moe_config: backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml index e797d8e71c9a..2eea897e2f58 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml index 1121d4e4b232..1a0d44fb2778 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml index 2a74eaeccfaa..82662456f032 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml index e8c2ded159ca..57d8e2ada29b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml index aa6194fedb46..87e34788d7c6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml index e797d8e71c9a..57b4b87fc72d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml index 1121d4e4b232..0d796e475135 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml index 2a74eaeccfaa..f6c41d8bbdf7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml index e8c2ded159ca..fdec025db8b9 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml index aa6194fedb46..8565e82e3623 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml index e797d8e71c9a..4773067517e2 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml index 1121d4e4b232..5e0d27c5ead3 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml index 2a74eaeccfaa..9b135c0a3215 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml index e8c2ded159ca..6874784b9f29 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml index aa6194fedb46..cc1d2d8ac986 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml index e797d8e71c9a..f7e46b17a369 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml index 1121d4e4b232..1b1b874c3e1b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml index 2a74eaeccfaa..28a7f3d17c69 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml index e8c2ded159ca..8036e7439962 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml index aa6194fedb46..12289904eda8 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml index e797d8e71c9a..7ccdc4ae1146 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml index 1121d4e4b232..ea6a93ba64bb 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml index 2a74eaeccfaa..a0149f2ab5c5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml index e8c2ded159ca..3ae56a300ad6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml index aa6194fedb46..c18bc3c7581c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml index e797d8e71c9a..e88b4e05fe8a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml index 1121d4e4b232..95b8e207330d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml index 2a74eaeccfaa..c35b691a81c5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml index e8c2ded159ca..ce0f7c27577c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml index aa6194fedb46..344166bc325b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml index e797d8e71c9a..4f895199b1f2 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml index 1121d4e4b232..ca549de3d28a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml index 2a74eaeccfaa..b87044bbc0ea 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml index e8c2ded159ca..9af104970e78 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml index aa6194fedb46..7440c3fcb7e7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml index e797d8e71c9a..b1d8a6eead07 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml index 1121d4e4b232..f8c7fec13a7a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml index 2a74eaeccfaa..f9cb8feb6956 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml index e8c2ded159ca..a9124d700757 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml index aa6194fedb46..7c2507ace7aa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml index e797d8e71c9a..7ccdc4ae1146 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml index 1121d4e4b232..ea6a93ba64bb 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml index 2a74eaeccfaa..a0149f2ab5c5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml index e8c2ded159ca..3ae56a300ad6 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml index aa6194fedb46..c18bc3c7581c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml index e797d8e71c9a..e88b4e05fe8a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml index 1121d4e4b232..95b8e207330d 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml index 2a74eaeccfaa..c35b691a81c5 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml index e8c2ded159ca..ce0f7c27577c 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml index aa6194fedb46..344166bc325b 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml index e797d8e71c9a..4f895199b1f2 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml index 1121d4e4b232..ca549de3d28a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml index 2a74eaeccfaa..b87044bbc0ea 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml index e8c2ded159ca..9af104970e78 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml index aa6194fedb46..7440c3fcb7e7 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml index e797d8e71c9a..b1d8a6eead07 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml index 1121d4e4b232..f8c7fec13a7a 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml index 2a74eaeccfaa..f9cb8feb6956 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml index e8c2ded159ca..a9124d700757 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml index aa6194fedb46..7c2507ace7aa 100644 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml @@ -13,3 +13,9 @@ moe_config: num_postprocess_workers: 4 print_iter_log: true stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 From b7298fd1cea964be725f5d500e75163fc8ab13f4 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 24 Nov 2025 22:08:09 -0800 Subject: [PATCH 17/39] move test to right place Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tests/unittest/{ => tools}/test_generate_config_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename tests/unittest/{ => tools}/test_generate_config_table.py (99%) diff --git a/tests/unittest/test_generate_config_table.py b/tests/unittest/tools/test_generate_config_table.py similarity index 99% rename from tests/unittest/test_generate_config_table.py rename to tests/unittest/tools/test_generate_config_table.py index a3ef0fca6607..d89067c222a3 100644 --- a/tests/unittest/test_generate_config_table.py +++ b/tests/unittest/tools/test_generate_config_table.py @@ -4,7 +4,7 @@ import unittest # Add scripts directory to path -REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../..")) +REPO_ROOT = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../..")) SCRIPTS_DIR = os.path.join(REPO_ROOT, "scripts") sys.path.insert(0, SCRIPTS_DIR) From 366fbd2d85ff2487407896c5a027f8ced8a44060 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Tue, 25 Nov 2025 14:24:00 -0800 Subject: [PATCH 18/39] swap columns Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide/comprehensive_table.rst | 690 +++++++++--------- scripts/generate_config_table.py | 16 +- 2 files changed, 352 insertions(+), 354 deletions(-) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index cd3c310e8b17..8ae4b1a8b9dd 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -8,134 +8,134 @@ .. list-table:: :width: 100% :header-rows: 1 - :widths: 12 15 20 25 15 13 + :widths: 12 15 15 13 20 25 * - GPU - Performance Profile + - ISL / OSL + - Concurrency - Config - Command - - Best ISL / OSL - - Best Concurrency * - B200_NVL - Min Latency - - `fp8_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 + - `fp8_b200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - - `fp8_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 + - `fp8_b200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - - `fp8_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 + - `fp8_b200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - - `fp8_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 + - `fp8_b200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - - `fp8_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 + - `fp8_b200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - - `fp8_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 + - `fp8_b200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - - `fp8_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 + - `fp8_b200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - - `fp8_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 + - `fp8_b200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - - `fp8_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 + - `fp8_b200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - - `fp8_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 + - `fp8_b200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - - `fp8_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 + - `fp8_h200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - - `fp8_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 + - `fp8_h200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - - `fp8_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 + - `fp8_h200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - - `fp8_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 + - `fp8_h200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - - `fp8_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 + - `fp8_h200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - - `fp8_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 + - `fp8_h200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - - `fp8_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 + - `fp8_h200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - - `fp8_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 + - `fp8_h200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - - `fp8_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 + - `fp8_h200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - - `fp8_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 + - `fp8_h200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` .. end-deepseek-ai/DeepSeek-R1-0528 @@ -149,182 +149,182 @@ .. list-table:: :width: 100% :header-rows: 1 - :widths: 12 15 20 25 15 13 + :widths: 12 15 15 13 20 25 * - GPU - Performance Profile + - ISL / OSL + - Concurrency - Config - Command - - Best ISL / OSL - - Best Concurrency * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` - 1024 / 1024 - 128 + - `fp4_b200_trt_1k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` - 1024 / 1024 - 128 + - `fp4_b200_trt_1k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` - 1024 / 1024 - 256 + - `fp4_b200_trt_1k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` - 1024 / 1024 - 256 + - `fp4_b200_trt_1k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_8k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` - 8192 / 1024 - 128 + - `fp4_b200_trt_8k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_8k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` - 8192 / 1024 - 128 + - `fp4_b200_trt_8k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_8k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` - 8192 / 1024 - 256 + - `fp4_b200_trt_8k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_8k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` - 8192 / 1024 - 256 + - `fp4_b200_trt_8k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` .. end-nvidia/DeepSeek-R1-0528-FP4-v2 @@ -338,733 +338,733 @@ .. list-table:: :width: 100% :header-rows: 1 - :widths: 12 15 20 25 15 13 + :widths: 12 15 15 13 20 25 * - GPU - Performance Profile + - ISL / OSL + - Concurrency - Config - Command - - Best ISL / OSL - - Best Concurrency * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_b200_trt_1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_b200_trt_1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_b200_trt_1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_b200_trt_1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_b200_trt_1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_b200_trt_1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_b200_trt_1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_b200_trt_1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_b200_trt_1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_b200_trt_1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_b200_trt_1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_b200_trt_1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_b200_trt_1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_b200_trt_1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_b200_trt_1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_b200_trt_1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_b200_trt_1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_b200_trt_1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_b200_trt_1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_b200_trt_1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_b200_trt_1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_b200_trt_1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_b200_trt_1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_b200_trt_1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_b200_trt_1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_b200_trt_1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_b200_trt_1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_b200_trt_1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_b200_trt_1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_b200_trt_1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_b200_trt_8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_b200_trt_8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_b200_trt_8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_b200_trt_8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_b200_trt_8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_b200_trt_8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` * - B200_NVL - Min Latency - - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_b200_trt_8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_b200_trt_8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` * - B200_NVL - Balanced - - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_b200_trt_8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_b200_trt_8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` * - B200_NVL - Max Throughput - - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_h200_trt_1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_h200_trt_1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_h200_trt_1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` - 1024 / 1024 - 4 + - `fp4_h200_trt_1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_h200_trt_1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_h200_trt_1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_h200_trt_1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` - 1024 / 1024 - 8 + - `fp4_h200_trt_1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_h200_trt_1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_h200_trt_1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_h200_trt_1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` - 1024 / 1024 - 16 + - `fp4_h200_trt_1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_h200_trt_1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_h200_trt_1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_h200_trt_1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` - 1024 / 1024 - 32 + - `fp4_h200_trt_1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_h200_trt_1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_h200_trt_1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_h200_trt_1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` - 1024 / 1024 - 64 + - `fp4_h200_trt_1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_h200_trt_1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_h200_trt_1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_h200_trt_1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` - 1024 / 8192 - 4 + - `fp4_h200_trt_1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_h200_trt_1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_h200_trt_1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_h200_trt_1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` - 1024 / 8192 - 8 + - `fp4_h200_trt_1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_h200_trt_1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_h200_trt_1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_h200_trt_1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` - 1024 / 8192 - 16 + - `fp4_h200_trt_1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_h200_trt_1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_h200_trt_1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_h200_trt_1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` - 1024 / 8192 - 32 + - `fp4_h200_trt_1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_h200_trt_1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_h200_trt_1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_h200_trt_1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` - 1024 / 8192 - 64 + - `fp4_h200_trt_1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_h200_trt_8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_h200_trt_8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_h200_trt_8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` - 8192 / 1024 - 4 + - `fp4_h200_trt_8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_h200_trt_8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_h200_trt_8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_h200_trt_8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` - 8192 / 1024 - 8 + - `fp4_h200_trt_8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_h200_trt_8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_h200_trt_8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_h200_trt_8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` * - H200_SXM - Min Latency - - `fp4_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` - 8192 / 1024 - 16 + - `fp4_h200_trt_8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_h200_trt_8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_h200_trt_8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_h200_trt_8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` * - H200_SXM - Balanced - - `fp4_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` - 8192 / 1024 - 32 + - `fp4_h200_trt_8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_h200_trt_8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_h200_trt_8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_h200_trt_8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` * - H200_SXM - Max Throughput - - `fp4_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` - 8192 / 1024 - 64 + - `fp4_h200_trt_8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` .. end-openai/gpt-oss-120b diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 15113f823308..77d2554bc2a6 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -70,15 +70,15 @@ def generate_rst(yaml_path, output_file=None): lines.append(".. list-table::") lines.append(" :width: 100%") lines.append(" :header-rows: 1") - # Widths: GPU, Perf Profile, Config, Command, Recommended Request Settings (ISL/OSL, Concurrency) - lines.append(" :widths: 12 15 20 25 15 13") + # Widths: GPU, Perf Profile, ISL/OSL, Concurrency, Config, Command + lines.append(" :widths: 12 15 15 13 20 25") lines.append("") lines.append(" * - GPU") lines.append(" - Performance Profile") + lines.append(" - ISL / OSL") + lines.append(" - Concurrency") lines.append(" - Config") lines.append(" - Command") - lines.append(" - Best ISL / OSL") - lines.append(" - Best Concurrency") # Process entries for this model subgroups = model_groups[model] @@ -126,9 +126,7 @@ def generate_rst(yaml_path, output_file=None): profile = "Balanced" full_config_path = os.path.join("tensorrt_llm/configure", config_path) - command = ( - f"trtllm-serve {model} --extra_llm_api_options ${{TRTLLM_DIR}}/{config_path}" - ) + command = f"trtllm-serve {model} --extra_llm_api_options ${{TRTLLM_DIR}}/{full_config_path}" config_filename = os.path.basename(full_config_path) @@ -143,10 +141,10 @@ def generate_rst(yaml_path, output_file=None): lines.append(f" * - {gpu}") lines.append(f" - {profile}") - lines.append(f" - {config_link}") - lines.append(f" - ``{command}``") lines.append(f" - {isl} / {osl}") lines.append(f" - {conc}") + lines.append(f" - {config_link}") + lines.append(f" - ``{command}``") lines.append("") # Space between tables lines.append(f".. end-{model}") From 3569f2bb46ced3d46594ea6e727a0c3691b2775f Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:34:47 -0800 Subject: [PATCH 19/39] modify wording Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide/comprehensive_table.rst | 17 ++++++++++ docs/source/deployment-guide/index.rst | 2 +- scripts/generate_config_table.py | 31 +++++++++++++++++++ 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 8ae4b1a8b9dd..360571b73b70 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -1,3 +1,20 @@ +.. note:: + + **Traffic Patterns**: The ISL (Input Sequence Length) and OSL (Output Sequence Length) + values in each configuration represent the **maximum supported values** for that config. + Requests exceeding these limits may result in errors. + + To handle requests with input sequences **longer than the configured ISL**, add the following + to your config file: + + .. code-block:: yaml + + enable_chunked_prefill: true + + This enables chunked prefill, which processes long input sequences in chunks rather than + requiring them to fit within a single prefill operation. Note that enabling chunked prefill + does **not** guarantee optimal performance—these configs are tuned for the specified ISL/OSL. + .. start-deepseek-ai/DeepSeek-R1-0528 .. _deepseek-ai/DeepSeek-R1-0528: diff --git a/docs/source/deployment-guide/index.rst b/docs/source/deployment-guide/index.rst index 60a224fbc418..b179fc34c7be 100644 --- a/docs/source/deployment-guide/index.rst +++ b/docs/source/deployment-guide/index.rst @@ -19,7 +19,7 @@ The TensorRT LLM Docker container makes these config files available at ``/app/t .. note:: - The configs here are specifically optimized for a target ISL/OSL (Input/Output Sequence Length) of 1024/1024. If your traffic pattern is different, you may benefit from additional tuning. In the future, we plan to provide more configs for a wider range of traffic patterns. + The configs here are specifically optimized for a target ISL/OSL (Input/Output Sequence Length) of 1024/1024. If your traffic pattern is different, refer to the :ref:`Comprehensive Configuration Database` section below which covers a larger set of traffic patterns and performance profiles. This table is designed to provide a straightforward starting point; for detailed model-specific deployment guides, check out the guides below. diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 77d2554bc2a6..c7fb43ac6bb4 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -45,6 +45,37 @@ def generate_rst(yaml_path, output_file=None): # Prepare output lines lines = [] + # Add introductory note about ISL/OSL constraints + lines.append(".. note::") + lines.append("") + lines.append( + " **Traffic Patterns**: The ISL (Input Sequence Length) and OSL (Output Sequence Length)" + ) + lines.append( + " values in each configuration represent the **maximum supported values** for that config." + ) + lines.append(" Requests exceeding these limits may result in errors.") + lines.append("") + lines.append( + " To handle requests with input sequences **longer than the configured ISL**, add the following" + ) + lines.append(" to your config file:") + lines.append("") + lines.append(" .. code-block:: yaml") + lines.append("") + lines.append(" enable_chunked_prefill: true") + lines.append("") + lines.append( + " This enables chunked prefill, which processes long input sequences in chunks rather than" + ) + lines.append( + " requiring them to fit within a single prefill operation. Note that enabling chunked prefill" + ) + lines.append( + " does **not** guarantee optimal performance—these configs are tuned for the specified ISL/OSL." + ) + lines.append("") + # Sort models alphabetically sorted_models = sorted(model_groups.keys()) From 9eeacbe678aea40fea49f5f4e207efbbd3828c21 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 1 Dec 2025 16:54:57 -0800 Subject: [PATCH 20/39] remove tps_* metrics from config db for now Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 4 +- .../configure/database/scenario_list.yaml | 336 ------------------ 2 files changed, 1 insertion(+), 339 deletions(-) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 47655e2a5f87..226e7c8255ab 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -1,5 +1,5 @@ from pathlib import Path -from typing import Any, Dict, Iterator, List, Optional +from typing import Any, Dict, Iterator, List import pandas as pd import yaml @@ -17,8 +17,6 @@ class RecipeConstraints(BaseModel): osl: int = Field(description="Output sequence length") concurrency: int = Field(description="Concurrency") config_path: str = Field(description="Configuration path") - tps_per_user: Optional[int] = Field(description="TPS per user", default=None) - tps_per_gpu: Optional[int] = Field(description="TPS per GPU", default=None) num_gpus: int = Field(description="Number of GPUs") def to_pandas_row(self) -> pd.Series: diff --git a/tensorrt_llm/configure/database/scenario_list.yaml b/tensorrt_llm/configure/database/scenario_list.yaml index dacb09069ccb..c918759403f1 100644 --- a/tensorrt_llm/configure/database/scenario_list.yaml +++ b/tensorrt_llm/configure/database/scenario_list.yaml @@ -4,8 +4,6 @@ osl: 1024 concurrency: 128 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -13,8 +11,6 @@ osl: 1024 concurrency: 16 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -22,8 +18,6 @@ osl: 1024 concurrency: 256 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -31,8 +25,6 @@ osl: 1024 concurrency: 32 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -40,8 +32,6 @@ osl: 1024 concurrency: 4 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -49,8 +39,6 @@ osl: 1024 concurrency: 64 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -58,8 +46,6 @@ osl: 1024 concurrency: 8 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -67,8 +53,6 @@ osl: 1024 concurrency: 128 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -76,8 +60,6 @@ osl: 1024 concurrency: 16 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -85,8 +67,6 @@ osl: 1024 concurrency: 256 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -94,8 +74,6 @@ osl: 1024 concurrency: 32 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -103,8 +81,6 @@ osl: 1024 concurrency: 4 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -112,8 +88,6 @@ osl: 1024 concurrency: 64 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -121,8 +95,6 @@ osl: 1024 concurrency: 8 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -130,8 +102,6 @@ osl: 1024 concurrency: 128 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -139,8 +109,6 @@ osl: 1024 concurrency: 16 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -148,8 +116,6 @@ osl: 1024 concurrency: 256 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -157,8 +123,6 @@ osl: 1024 concurrency: 32 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -166,8 +130,6 @@ osl: 1024 concurrency: 4 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -175,8 +137,6 @@ osl: 1024 concurrency: 64 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -184,8 +144,6 @@ osl: 1024 concurrency: 8 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -193,8 +151,6 @@ osl: 1024 concurrency: 128 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -202,8 +158,6 @@ osl: 1024 concurrency: 16 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -211,8 +165,6 @@ osl: 1024 concurrency: 256 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -220,8 +172,6 @@ osl: 1024 concurrency: 32 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -229,8 +179,6 @@ osl: 1024 concurrency: 4 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -238,8 +186,6 @@ osl: 1024 concurrency: 64 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL @@ -247,8 +193,6 @@ osl: 1024 concurrency: 8 config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -256,8 +200,6 @@ osl: 1024 concurrency: 16 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -265,8 +207,6 @@ osl: 1024 concurrency: 32 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -274,8 +214,6 @@ osl: 1024 concurrency: 4 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -283,8 +221,6 @@ osl: 1024 concurrency: 64 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -292,8 +228,6 @@ osl: 1024 concurrency: 8 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -301,8 +235,6 @@ osl: 1024 concurrency: 16 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -310,8 +242,6 @@ osl: 1024 concurrency: 32 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -319,8 +249,6 @@ osl: 1024 concurrency: 4 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -328,8 +256,6 @@ osl: 1024 concurrency: 64 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL @@ -337,8 +263,6 @@ osl: 1024 concurrency: 8 config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -346,8 +270,6 @@ osl: 1024 concurrency: 16 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -355,8 +277,6 @@ osl: 1024 concurrency: 32 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -364,8 +284,6 @@ osl: 1024 concurrency: 4 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -373,8 +291,6 @@ osl: 1024 concurrency: 64 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -382,8 +298,6 @@ osl: 1024 concurrency: 8 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -391,8 +305,6 @@ osl: 1024 concurrency: 16 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -400,8 +312,6 @@ osl: 1024 concurrency: 32 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -409,8 +319,6 @@ osl: 1024 concurrency: 4 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -418,8 +326,6 @@ osl: 1024 concurrency: 64 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM @@ -427,8 +333,6 @@ osl: 1024 concurrency: 8 config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -436,8 +340,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -445,8 +347,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -454,8 +354,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -463,8 +361,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -472,8 +368,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -481,8 +375,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -490,8 +382,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -499,8 +389,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -508,8 +396,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -517,8 +403,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -526,8 +410,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -535,8 +417,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -544,8 +424,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -553,8 +431,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -562,8 +438,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -571,8 +445,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -580,8 +452,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -589,8 +459,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -598,8 +466,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -607,8 +473,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -616,8 +480,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -625,8 +487,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -634,8 +494,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -643,8 +501,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -652,8 +508,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -661,8 +515,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -670,8 +522,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -679,8 +529,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -688,8 +536,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -697,8 +543,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -706,8 +550,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -715,8 +557,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -724,8 +564,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -733,8 +571,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -742,8 +578,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -751,8 +585,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -760,8 +592,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -769,8 +599,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -778,8 +606,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -787,8 +613,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -796,8 +620,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -805,8 +627,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -814,8 +634,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -823,8 +641,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -832,8 +648,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -841,8 +655,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -850,8 +662,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -859,8 +669,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -868,8 +676,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -877,8 +683,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -886,8 +690,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -895,8 +697,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -904,8 +704,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -913,8 +711,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -922,8 +718,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -931,8 +725,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -940,8 +732,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -949,8 +739,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -958,8 +746,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL @@ -967,8 +753,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -976,8 +760,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -985,8 +767,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -994,8 +774,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1003,8 +781,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1012,8 +788,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1021,8 +795,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1030,8 +802,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1039,8 +809,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1048,8 +816,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1057,8 +823,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1066,8 +830,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1075,8 +837,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1084,8 +844,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1093,8 +851,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1102,8 +858,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1111,8 +865,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1120,8 +872,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1129,8 +879,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1138,8 +886,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1147,8 +893,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1156,8 +900,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1165,8 +907,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1174,8 +914,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1183,8 +921,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1192,8 +928,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1201,8 +935,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1210,8 +942,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1219,8 +949,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1228,8 +956,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1237,8 +963,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1246,8 +970,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1255,8 +977,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1264,8 +984,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1273,8 +991,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1282,8 +998,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1291,8 +1005,6 @@ osl: 8192 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1300,8 +1012,6 @@ osl: 8192 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1309,8 +1019,6 @@ osl: 8192 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1318,8 +1026,6 @@ osl: 8192 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1327,8 +1033,6 @@ osl: 8192 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1336,8 +1040,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1345,8 +1047,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1354,8 +1054,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1363,8 +1061,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1372,8 +1068,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1381,8 +1075,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1390,8 +1082,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1399,8 +1089,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1408,8 +1096,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1417,8 +1103,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1426,8 +1110,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1435,8 +1117,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1444,8 +1124,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1453,8 +1131,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1462,8 +1138,6 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1471,8 +1145,6 @@ osl: 1024 concurrency: 16 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1480,8 +1152,6 @@ osl: 1024 concurrency: 32 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1489,8 +1159,6 @@ osl: 1024 concurrency: 4 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1498,8 +1166,6 @@ osl: 1024 concurrency: 64 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM @@ -1507,6 +1173,4 @@ osl: 1024 concurrency: 8 config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml - tps_per_user: null - tps_per_gpu: null num_gpus: 8 From 13a6b6a018cef7c1983d3f50acd0a1fe9f113d32 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Tue, 2 Dec 2025 09:55:07 -0800 Subject: [PATCH 21/39] final tweaks Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- docs/source/deployment-guide/index.rst | 2 +- tests/unittest/tools/test_generate_config_table.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/source/deployment-guide/index.rst b/docs/source/deployment-guide/index.rst index b179fc34c7be..f4f090aeac66 100644 --- a/docs/source/deployment-guide/index.rst +++ b/docs/source/deployment-guide/index.rst @@ -103,6 +103,6 @@ The deployment guides below provide more detailed instructions for serving speci Comprehensive Configuration Database ------------------------------------ -The table below lists all available pre-configured model scenarios in the TensorRT-LLM configuration database. Each row represents a specific model, GPU, and performance profile combination with recommended request settings. +The table below lists all available pre-configured model scenarios in the TensorRT LLM configuration database. Each row represents a specific model, GPU, and performance profile combination with recommended request settings. .. include:: comprehensive_table.rst diff --git a/tests/unittest/tools/test_generate_config_table.py b/tests/unittest/tools/test_generate_config_table.py index d89067c222a3..a19364641b3e 100644 --- a/tests/unittest/tools/test_generate_config_table.py +++ b/tests/unittest/tools/test_generate_config_table.py @@ -8,7 +8,6 @@ SCRIPTS_DIR = os.path.join(REPO_ROOT, "scripts") sys.path.insert(0, SCRIPTS_DIR) -# Try to import generate_rst, fail gracefully if dependencies (like yaml) are missing from generate_config_table import generate_rst # noqa: E402 From 88e4da352f84c58ba0fab18ec8c3d698948d29c1 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:03:25 -0800 Subject: [PATCH 22/39] include num_gpu information in table Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide/comprehensive_table.rst | 276 +++++++++--------- scripts/generate_config_table.py | 4 +- 2 files changed, 141 insertions(+), 139 deletions(-) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 360571b73b70..561d2d15efcb 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -33,121 +33,121 @@ - Concurrency - Config - Command - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 4 - `fp8_b200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 8 - `fp8_b200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 16 - `fp8_b200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` - * - B200_NVL + * - 8xB200_NVL - Balanced - 1024 / 1024 - 32 - `fp8_b200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` - * - B200_NVL + * - 8xB200_NVL - Max Throughput - 1024 / 1024 - 64 - `fp8_b200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 4 - `fp8_b200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 8 - `fp8_b200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 16 - `fp8_b200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` - * - B200_NVL + * - 8xB200_NVL - Balanced - 8192 / 1024 - 32 - `fp8_b200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` - * - B200_NVL + * - 8xB200_NVL - Max Throughput - 8192 / 1024 - 64 - `fp8_b200_trt_8k1k_tp8_conc64.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 1024 - 4 - `fp8_h200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 1024 - 8 - `fp8_h200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 1024 - 16 - `fp8_h200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` - * - H200_SXM + * - 8xH200_SXM - Balanced - 1024 / 1024 - 32 - `fp8_h200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` - * - H200_SXM + * - 8xH200_SXM - Max Throughput - 1024 / 1024 - 64 - `fp8_h200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 8192 / 1024 - 4 - `fp8_h200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 8192 / 1024 - 8 - `fp8_h200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 8192 / 1024 - 16 - `fp8_h200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` - * - H200_SXM + * - 8xH200_SXM - Balanced - 8192 / 1024 - 32 - `fp8_h200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` - * - H200_SXM + * - 8xH200_SXM - Max Throughput - 8192 / 1024 - 64 @@ -174,169 +174,169 @@ - Concurrency - Config - Command - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 4 - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 4 - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 8 - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 8 - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 16 - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 16 - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 32 - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 32 - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 64 - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 64 - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` - * - B200_NVL + * - 4xB200_NVL - Balanced - 1024 / 1024 - 128 - `fp4_b200_trt_1k1k_tp4_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` - * - B200_NVL + * - 8xB200_NVL - Balanced - 1024 / 1024 - 128 - `fp4_b200_trt_1k1k_tp8_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` - * - B200_NVL + * - 4xB200_NVL - Max Throughput - 1024 / 1024 - 256 - `fp4_b200_trt_1k1k_tp4_conc256.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` - * - B200_NVL + * - 8xB200_NVL - Max Throughput - 1024 / 1024 - 256 - `fp4_b200_trt_1k1k_tp8_conc256.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 4 - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 4 - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 8 - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 8 - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 16 - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 16 - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 32 - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 32 - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 64 - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 64 - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` - * - B200_NVL + * - 4xB200_NVL - Balanced - 8192 / 1024 - 128 - `fp4_b200_trt_8k1k_tp4_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` - * - B200_NVL + * - 8xB200_NVL - Balanced - 8192 / 1024 - 128 - `fp4_b200_trt_8k1k_tp8_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` - * - B200_NVL + * - 4xB200_NVL - Max Throughput - 8192 / 1024 - 256 - `fp4_b200_trt_8k1k_tp4_conc256.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` - * - B200_NVL + * - 8xB200_NVL - Max Throughput - 8192 / 1024 - 256 @@ -369,19 +369,19 @@ - 4 - `fp4_b200_trt_1k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 1024 / 1024 - 4 - `fp4_b200_trt_1k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 4 - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 4 @@ -393,19 +393,19 @@ - 8 - `fp4_b200_trt_1k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 1024 / 1024 - 8 - `fp4_b200_trt_1k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 8 - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 8 @@ -417,19 +417,19 @@ - 16 - `fp4_b200_trt_1k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 1024 / 1024 - 16 - `fp4_b200_trt_1k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 1024 - 16 - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 1024 - 16 @@ -441,19 +441,19 @@ - 32 - `fp4_b200_trt_1k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` - * - B200_NVL + * - 2xB200_NVL - Balanced - 1024 / 1024 - 32 - `fp4_b200_trt_1k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` - * - B200_NVL + * - 4xB200_NVL - Balanced - 1024 / 1024 - 32 - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` - * - B200_NVL + * - 8xB200_NVL - Balanced - 1024 / 1024 - 32 @@ -465,19 +465,19 @@ - 64 - `fp4_b200_trt_1k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` - * - B200_NVL + * - 2xB200_NVL - Max Throughput - 1024 / 1024 - 64 - `fp4_b200_trt_1k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` - * - B200_NVL + * - 4xB200_NVL - Max Throughput - 1024 / 1024 - 64 - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` - * - B200_NVL + * - 8xB200_NVL - Max Throughput - 1024 / 1024 - 64 @@ -489,19 +489,19 @@ - 4 - `fp4_b200_trt_1k8k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 1024 / 8192 - 4 - `fp4_b200_trt_1k8k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 8192 - 4 - `fp4_b200_trt_1k8k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 8192 - 4 @@ -513,19 +513,19 @@ - 8 - `fp4_b200_trt_1k8k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 1024 / 8192 - 8 - `fp4_b200_trt_1k8k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 8192 - 8 - `fp4_b200_trt_1k8k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 8192 - 8 @@ -537,19 +537,19 @@ - 16 - `fp4_b200_trt_1k8k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 1024 / 8192 - 16 - `fp4_b200_trt_1k8k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 1024 / 8192 - 16 - `fp4_b200_trt_1k8k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 1024 / 8192 - 16 @@ -561,19 +561,19 @@ - 32 - `fp4_b200_trt_1k8k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` - * - B200_NVL + * - 2xB200_NVL - Balanced - 1024 / 8192 - 32 - `fp4_b200_trt_1k8k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` - * - B200_NVL + * - 4xB200_NVL - Balanced - 1024 / 8192 - 32 - `fp4_b200_trt_1k8k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` - * - B200_NVL + * - 8xB200_NVL - Balanced - 1024 / 8192 - 32 @@ -585,19 +585,19 @@ - 64 - `fp4_b200_trt_1k8k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` - * - B200_NVL + * - 2xB200_NVL - Max Throughput - 1024 / 8192 - 64 - `fp4_b200_trt_1k8k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` - * - B200_NVL + * - 4xB200_NVL - Max Throughput - 1024 / 8192 - 64 - `fp4_b200_trt_1k8k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` - * - B200_NVL + * - 8xB200_NVL - Max Throughput - 1024 / 8192 - 64 @@ -609,19 +609,19 @@ - 4 - `fp4_b200_trt_8k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 8192 / 1024 - 4 - `fp4_b200_trt_8k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 4 - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 4 @@ -633,19 +633,19 @@ - 8 - `fp4_b200_trt_8k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 8192 / 1024 - 8 - `fp4_b200_trt_8k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 8 - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 8 @@ -657,19 +657,19 @@ - 16 - `fp4_b200_trt_8k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` - * - B200_NVL + * - 2xB200_NVL - Min Latency - 8192 / 1024 - 16 - `fp4_b200_trt_8k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` - * - B200_NVL + * - 4xB200_NVL - Min Latency - 8192 / 1024 - 16 - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` - * - B200_NVL + * - 8xB200_NVL - Min Latency - 8192 / 1024 - 16 @@ -681,19 +681,19 @@ - 32 - `fp4_b200_trt_8k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` - * - B200_NVL + * - 2xB200_NVL - Balanced - 8192 / 1024 - 32 - `fp4_b200_trt_8k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` - * - B200_NVL + * - 4xB200_NVL - Balanced - 8192 / 1024 - 32 - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` - * - B200_NVL + * - 8xB200_NVL - Balanced - 8192 / 1024 - 32 @@ -705,19 +705,19 @@ - 64 - `fp4_b200_trt_8k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` - * - B200_NVL + * - 2xB200_NVL - Max Throughput - 8192 / 1024 - 64 - `fp4_b200_trt_8k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` - * - B200_NVL + * - 4xB200_NVL - Max Throughput - 8192 / 1024 - 64 - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` - * - B200_NVL + * - 8xB200_NVL - Max Throughput - 8192 / 1024 - 64 @@ -729,19 +729,19 @@ - 4 - `fp4_h200_trt_1k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 1024 / 1024 - 4 - `fp4_h200_trt_1k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 1024 / 1024 - 4 - `fp4_h200_trt_1k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 1024 - 4 @@ -753,19 +753,19 @@ - 8 - `fp4_h200_trt_1k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 1024 / 1024 - 8 - `fp4_h200_trt_1k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 1024 / 1024 - 8 - `fp4_h200_trt_1k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 1024 - 8 @@ -777,19 +777,19 @@ - 16 - `fp4_h200_trt_1k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 1024 / 1024 - 16 - `fp4_h200_trt_1k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 1024 / 1024 - 16 - `fp4_h200_trt_1k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 1024 - 16 @@ -801,19 +801,19 @@ - 32 - `fp4_h200_trt_1k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` - * - H200_SXM + * - 2xH200_SXM - Balanced - 1024 / 1024 - 32 - `fp4_h200_trt_1k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` - * - H200_SXM + * - 4xH200_SXM - Balanced - 1024 / 1024 - 32 - `fp4_h200_trt_1k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` - * - H200_SXM + * - 8xH200_SXM - Balanced - 1024 / 1024 - 32 @@ -825,19 +825,19 @@ - 64 - `fp4_h200_trt_1k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` - * - H200_SXM + * - 2xH200_SXM - Max Throughput - 1024 / 1024 - 64 - `fp4_h200_trt_1k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` - * - H200_SXM + * - 4xH200_SXM - Max Throughput - 1024 / 1024 - 64 - `fp4_h200_trt_1k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` - * - H200_SXM + * - 8xH200_SXM - Max Throughput - 1024 / 1024 - 64 @@ -849,19 +849,19 @@ - 4 - `fp4_h200_trt_1k8k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 1024 / 8192 - 4 - `fp4_h200_trt_1k8k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 1024 / 8192 - 4 - `fp4_h200_trt_1k8k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 8192 - 4 @@ -873,19 +873,19 @@ - 8 - `fp4_h200_trt_1k8k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 1024 / 8192 - 8 - `fp4_h200_trt_1k8k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 1024 / 8192 - 8 - `fp4_h200_trt_1k8k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 8192 - 8 @@ -897,19 +897,19 @@ - 16 - `fp4_h200_trt_1k8k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 1024 / 8192 - 16 - `fp4_h200_trt_1k8k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 1024 / 8192 - 16 - `fp4_h200_trt_1k8k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 1024 / 8192 - 16 @@ -921,19 +921,19 @@ - 32 - `fp4_h200_trt_1k8k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` - * - H200_SXM + * - 2xH200_SXM - Balanced - 1024 / 8192 - 32 - `fp4_h200_trt_1k8k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` - * - H200_SXM + * - 4xH200_SXM - Balanced - 1024 / 8192 - 32 - `fp4_h200_trt_1k8k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` - * - H200_SXM + * - 8xH200_SXM - Balanced - 1024 / 8192 - 32 @@ -945,19 +945,19 @@ - 64 - `fp4_h200_trt_1k8k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` - * - H200_SXM + * - 2xH200_SXM - Max Throughput - 1024 / 8192 - 64 - `fp4_h200_trt_1k8k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` - * - H200_SXM + * - 4xH200_SXM - Max Throughput - 1024 / 8192 - 64 - `fp4_h200_trt_1k8k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` - * - H200_SXM + * - 8xH200_SXM - Max Throughput - 1024 / 8192 - 64 @@ -969,19 +969,19 @@ - 4 - `fp4_h200_trt_8k1k_tp1_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 8192 / 1024 - 4 - `fp4_h200_trt_8k1k_tp2_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 8192 / 1024 - 4 - `fp4_h200_trt_8k1k_tp4_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 8192 / 1024 - 4 @@ -993,19 +993,19 @@ - 8 - `fp4_h200_trt_8k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 8192 / 1024 - 8 - `fp4_h200_trt_8k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 8192 / 1024 - 8 - `fp4_h200_trt_8k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 8192 / 1024 - 8 @@ -1017,19 +1017,19 @@ - 16 - `fp4_h200_trt_8k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` - * - H200_SXM + * - 2xH200_SXM - Min Latency - 8192 / 1024 - 16 - `fp4_h200_trt_8k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` - * - H200_SXM + * - 4xH200_SXM - Min Latency - 8192 / 1024 - 16 - `fp4_h200_trt_8k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` - * - H200_SXM + * - 8xH200_SXM - Min Latency - 8192 / 1024 - 16 @@ -1041,19 +1041,19 @@ - 32 - `fp4_h200_trt_8k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` - * - H200_SXM + * - 2xH200_SXM - Balanced - 8192 / 1024 - 32 - `fp4_h200_trt_8k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` - * - H200_SXM + * - 4xH200_SXM - Balanced - 8192 / 1024 - 32 - `fp4_h200_trt_8k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` - * - H200_SXM + * - 8xH200_SXM - Balanced - 8192 / 1024 - 32 @@ -1065,19 +1065,19 @@ - 64 - `fp4_h200_trt_8k1k_tp1_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` - * - H200_SXM + * - 2xH200_SXM - Max Throughput - 8192 / 1024 - 64 - `fp4_h200_trt_8k1k_tp2_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` - * - H200_SXM + * - 4xH200_SXM - Max Throughput - 8192 / 1024 - 64 - `fp4_h200_trt_8k1k_tp4_conc64.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` - * - H200_SXM + * - 8xH200_SXM - Max Throughput - 8192 / 1024 - 64 diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index c7fb43ac6bb4..9a069231ca05 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -131,6 +131,8 @@ def generate_rst(yaml_path, output_file=None): for entry in entries: gpu = entry.get("gpu", "N/A") + num_gpus = entry.get("num_gpus", 1) + gpu_display = f"{num_gpus}x{gpu}" if num_gpus and num_gpus > 1 else gpu isl = entry.get("isl", "N/A") osl = entry.get("osl", "N/A") conc = int(entry.get("concurrency", 0)) @@ -170,7 +172,7 @@ def generate_rst(yaml_path, output_file=None): github_url = f"https://github.com/NVIDIA/TensorRT-LLM/blob/main/{full_config_path}" config_link = f"`{config_filename} <{github_url}>`_" - lines.append(f" * - {gpu}") + lines.append(f" * - {gpu_display}") lines.append(f" - {profile}") lines.append(f" - {isl} / {osl}") lines.append(f" - {conc}") From 3b7b13635cded69b2be01d0dbbfc2492570360f0 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Tue, 2 Dec 2025 11:24:27 -0800 Subject: [PATCH 23/39] add safe load yaml to make precommit happy Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tensorrt_llm/configure/database.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 226e7c8255ab..4aea70bf7dac 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -30,7 +30,7 @@ def to_pandas_row(self) -> pd.Series: def load_config(self) -> Dict[str, Any]: """Load configuration from the configuration path.""" with open(self.config_path, "r") as f: - return Recipe(**yaml.load(f, Loader=yaml.FullLoader)) + return Recipe(**yaml.safe_load(f, Loader=yaml.FullLoader)) class Recipe(BaseModel): @@ -53,7 +53,7 @@ def from_yaml(cls, yaml_path: Path) -> "RecipeList": RecipeList: Recipe list object. """ with open(yaml_path, "r") as f: - data = yaml.load(f, Loader=yaml.FullLoader) + data = yaml.safe_load(f, Loader=yaml.FullLoader) return cls(data) def to_pandas_dataframe(self) -> pd.DataFrame: From cec62fcd976aee517620fcf8141c7788e508b3f1 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:11:40 -0800 Subject: [PATCH 24/39] fix env_override bug Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tensorrt_llm/llmapi/llm_args.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 9f154c53f63e..d69436c7e9d2 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -1959,10 +1959,10 @@ class BaseLlmArgs(StrictBaseModel): status="prototype", ) - env_overrides: Optional[Dict[str, str]] = Field( + env_overrides: Optional[Dict[str, Any]] = Field( default=None, description= - "[EXPERIMENTAL] Environment variable overrides. NOTE: import-time-cached env vars in the code won’t update unless the code fetches them from os.environ on demand.", + "[EXPERIMENTAL] Environment variable overrides. NOTE: import-time-cached env vars in the code won't update unless the code fetches them from os.environ on demand.", status="prototype") _parallel_config: Optional[_ParallelConfig] = PrivateAttr(default=None) From 83e1c7bb90f0c583ab1e172b900f5161924777d1 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Tue, 2 Dec 2025 16:18:37 -0800 Subject: [PATCH 25/39] fix env_override bug Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tensorrt_llm/llmapi/llm_args.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index d69436c7e9d2..e62d7f7d0399 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -1959,12 +1959,20 @@ class BaseLlmArgs(StrictBaseModel): status="prototype", ) - env_overrides: Optional[Dict[str, Any]] = Field( + env_overrides: Optional[Dict[str, str]] = Field( default=None, description= "[EXPERIMENTAL] Environment variable overrides. NOTE: import-time-cached env vars in the code won't update unless the code fetches them from os.environ on demand.", status="prototype") + @field_validator('env_overrides', mode='before') + @classmethod + def coerce_env_overrides_to_str(cls, v): + """Coerce env_overrides values to strings for os.environ compatibility.""" + if v is None: + return v + return {str(k): str(val) for k, val in v.items()} + _parallel_config: Optional[_ParallelConfig] = PrivateAttr(default=None) _model_format: Optional[_ModelFormatKind] = PrivateAttr(default=None) _speculative_model: Optional[str] = PrivateAttr(default=None) From c9f919a0a43df82cace0471e5fe61a439cf5e481 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Sun, 7 Dec 2025 23:11:11 -0800 Subject: [PATCH 26/39] simplify naming of files Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../run-benchmark-with-trtllm-serve.md | 2 +- .../deployment-guide/comprehensive_table.rst | 672 +++++++++--------- ...loyment-guide-for-deepseek-r1-on-trtllm.md | 2 +- .../deployment-guide-for-gpt-oss-on-trtllm.md | 2 +- ...oyment-guide-for-llama3.3-70b-on-trtllm.md | 2 +- ...oyment-guide-for-llama4-scout-on-trtllm.md | 2 +- docs/source/quick-start-guide.md | 2 +- scripts/generate_config_table.py | 11 +- ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml | 18 - ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml | 18 - ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml | 18 - ..._r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml | 18 - ...k_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml | 18 - ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml | 18 - ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml | 18 - ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml | 18 - ..._r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml | 22 - ...k_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml | 18 - ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml | 18 - ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml | 18 - ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml | 18 - ..._r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml | 18 - ...k_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml | 18 - ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml | 18 - ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml | 18 - ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml | 18 - ..._r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml | 22 - ...k_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml | 18 - ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml | 18 - ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml | 18 - ..._fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml | 22 - ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml | 18 - ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml | 18 - ...8_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml | 18 - ...28_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml | 18 - ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml | 18 - ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml | 18 - ..._fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml | 22 - ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml | 18 - ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml | 18 - ...8_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml | 18 - ...28_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml | 18 - ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml | 22 - ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml | 18 - ..._fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml | 22 - ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml | 18 - ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml | 18 - ...8_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml | 22 - ...28_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml | 18 - ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml | 22 - ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml | 18 - ..._fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml | 22 - ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml | 18 - ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml | 18 - ...8_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml | 22 - ...28_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml | 18 - ...oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml | 22 - ...oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml | 22 - ..._oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml | 22 - ...oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml | 21 - ...oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml | 21 - ..._oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml | 21 - .../configure/database/scenario_list.yaml | 336 ++++----- 177 files changed, 511 insertions(+), 4004 deletions(-) delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml delete mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml diff --git a/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md b/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md index 04e188ec0f8d..81c296fe9db3 100644 --- a/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md +++ b/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md @@ -25,7 +25,7 @@ TensorRT LLM distributes the pre-built container on [NGC Catalog](https://catalo You can launch the container using the following command: ```bash -docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:x.y.z +docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 ``` diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 561d2d15efcb..38a7476300f6 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -37,122 +37,122 @@ - Min Latency - 1024 / 1024 - 4 - - `fp8_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml`` + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 8 - - `fp8_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml`` + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 16 - - `fp8_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml`` + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml`` * - 8xB200_NVL - Balanced - 1024 / 1024 - 32 - - `fp8_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml`` + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml`` * - 8xB200_NVL - Max Throughput - 1024 / 1024 - 64 - - `fp8_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml`` + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 4 - - `fp8_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml`` + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 8 - - `fp8_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml`` + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 16 - - `fp8_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml`` + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml`` * - 8xB200_NVL - Balanced - 8192 / 1024 - 32 - - `fp8_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml`` + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml`` * - 8xB200_NVL - Max Throughput - 8192 / 1024 - 64 - - `fp8_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml`` + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 1024 - 4 - - `fp8_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml`` + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 1024 - 8 - - `fp8_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml`` + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 1024 - 16 - - `fp8_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml`` + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml`` * - 8xH200_SXM - Balanced - 1024 / 1024 - 32 - - `fp8_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml`` + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml`` * - 8xH200_SXM - Max Throughput - 1024 / 1024 - 64 - - `fp8_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml`` + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml`` * - 8xH200_SXM - Min Latency - 8192 / 1024 - 4 - - `fp8_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml`` + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml`` * - 8xH200_SXM - Min Latency - 8192 / 1024 - 8 - - `fp8_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml`` + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml`` * - 8xH200_SXM - Min Latency - 8192 / 1024 - 16 - - `fp8_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml`` + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml`` * - 8xH200_SXM - Balanced - 8192 / 1024 - 32 - - `fp8_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml`` + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml`` * - 8xH200_SXM - Max Throughput - 8192 / 1024 - 64 - - `fp8_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml`` + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml`` .. end-deepseek-ai/DeepSeek-R1-0528 @@ -178,170 +178,170 @@ - Min Latency - 1024 / 1024 - 4 - - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + - `1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 4 - - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 1024 - 8 - - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + - `1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 8 - - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 1024 - 16 - - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + - `1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 16 - - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 1024 - 32 - - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + - `1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 32 - - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 1024 - 64 - - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + - `1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 64 - - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml`` * - 4xB200_NVL - Balanced - 1024 / 1024 - 128 - - `fp4_b200_trt_1k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml`` + - `1k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml`` * - 8xB200_NVL - Balanced - 1024 / 1024 - 128 - - `fp4_b200_trt_1k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml`` + - `1k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml`` * - 4xB200_NVL - Max Throughput - 1024 / 1024 - 256 - - `fp4_b200_trt_1k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml`` + - `1k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml`` * - 8xB200_NVL - Max Throughput - 1024 / 1024 - 256 - - `fp4_b200_trt_1k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml`` + - `1k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 4 - - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + - `8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 4 - - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 8 - - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + - `8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 8 - - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 16 - - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + - `8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 16 - - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 32 - - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + - `8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 32 - - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 64 - - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + - `8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 64 - - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml`` * - 4xB200_NVL - Balanced - 8192 / 1024 - 128 - - `fp4_b200_trt_8k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml`` + - `8k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml`` * - 8xB200_NVL - Balanced - 8192 / 1024 - 128 - - `fp4_b200_trt_8k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml`` + - `8k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml`` * - 4xB200_NVL - Max Throughput - 8192 / 1024 - 256 - - `fp4_b200_trt_8k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml`` + - `8k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml`` * - 8xB200_NVL - Max Throughput - 8192 / 1024 - 256 - - `fp4_b200_trt_8k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml`` + - `8k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml`` .. end-nvidia/DeepSeek-R1-0528-FP4-v2 @@ -367,721 +367,721 @@ - Min Latency - 1024 / 1024 - 4 - - `fp4_b200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml`` + - `1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml`` * - 2xB200_NVL - Min Latency - 1024 / 1024 - 4 - - `fp4_b200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml`` + - `1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 1024 - 4 - - `fp4_b200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml`` + - `1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 4 - - `fp4_b200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml`` + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 8 - - `fp4_b200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml`` + - `1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml`` * - 2xB200_NVL - Min Latency - 1024 / 1024 - 8 - - `fp4_b200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml`` + - `1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 1024 - 8 - - `fp4_b200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml`` + - `1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 8 - - `fp4_b200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml`` + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 1024 - 16 - - `fp4_b200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml`` + - `1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml`` * - 2xB200_NVL - Min Latency - 1024 / 1024 - 16 - - `fp4_b200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml`` + - `1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 1024 - 16 - - `fp4_b200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml`` + - `1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 1024 - 16 - - `fp4_b200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml`` + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 1024 / 1024 - 32 - - `fp4_b200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml`` + - `1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml`` * - 2xB200_NVL - Balanced - 1024 / 1024 - 32 - - `fp4_b200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml`` + - `1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml`` * - 4xB200_NVL - Balanced - 1024 / 1024 - 32 - - `fp4_b200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml`` + - `1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml`` * - 8xB200_NVL - Balanced - 1024 / 1024 - 32 - - `fp4_b200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml`` + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 1024 / 1024 - 64 - - `fp4_b200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml`` + - `1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml`` * - 2xB200_NVL - Max Throughput - 1024 / 1024 - 64 - - `fp4_b200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml`` + - `1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml`` * - 4xB200_NVL - Max Throughput - 1024 / 1024 - 64 - - `fp4_b200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml`` + - `1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml`` * - 8xB200_NVL - Max Throughput - 1024 / 1024 - 64 - - `fp4_b200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml`` + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 4 - - `fp4_b200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml`` + - `1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml`` * - 2xB200_NVL - Min Latency - 1024 / 8192 - 4 - - `fp4_b200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml`` + - `1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 8192 - 4 - - `fp4_b200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml`` + - `1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 8192 - 4 - - `fp4_b200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml`` + - `1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 8 - - `fp4_b200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml`` + - `1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml`` * - 2xB200_NVL - Min Latency - 1024 / 8192 - 8 - - `fp4_b200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml`` + - `1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 8192 - 8 - - `fp4_b200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml`` + - `1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 8192 - 8 - - `fp4_b200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml`` + - `1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 1024 / 8192 - 16 - - `fp4_b200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml`` + - `1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml`` * - 2xB200_NVL - Min Latency - 1024 / 8192 - 16 - - `fp4_b200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml`` + - `1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml`` * - 4xB200_NVL - Min Latency - 1024 / 8192 - 16 - - `fp4_b200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml`` + - `1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml`` * - 8xB200_NVL - Min Latency - 1024 / 8192 - 16 - - `fp4_b200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml`` + - `1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 1024 / 8192 - 32 - - `fp4_b200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml`` + - `1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml`` * - 2xB200_NVL - Balanced - 1024 / 8192 - 32 - - `fp4_b200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml`` + - `1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml`` * - 4xB200_NVL - Balanced - 1024 / 8192 - 32 - - `fp4_b200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml`` + - `1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml`` * - 8xB200_NVL - Balanced - 1024 / 8192 - 32 - - `fp4_b200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml`` + - `1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 1024 / 8192 - 64 - - `fp4_b200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml`` + - `1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml`` * - 2xB200_NVL - Max Throughput - 1024 / 8192 - 64 - - `fp4_b200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml`` + - `1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml`` * - 4xB200_NVL - Max Throughput - 1024 / 8192 - 64 - - `fp4_b200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml`` + - `1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml`` * - 8xB200_NVL - Max Throughput - 1024 / 8192 - 64 - - `fp4_b200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml`` + - `1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 4 - - `fp4_b200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml`` + - `8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml`` * - 2xB200_NVL - Min Latency - 8192 / 1024 - 4 - - `fp4_b200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml`` + - `8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 4 - - `fp4_b200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml`` + - `8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 4 - - `fp4_b200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml`` + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 8 - - `fp4_b200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml`` + - `8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml`` * - 2xB200_NVL - Min Latency - 8192 / 1024 - 8 - - `fp4_b200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml`` + - `8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 8 - - `fp4_b200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml`` + - `8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 8 - - `fp4_b200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml`` + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml`` * - B200_NVL - Min Latency - 8192 / 1024 - 16 - - `fp4_b200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml`` + - `8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml`` * - 2xB200_NVL - Min Latency - 8192 / 1024 - 16 - - `fp4_b200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml`` + - `8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml`` * - 4xB200_NVL - Min Latency - 8192 / 1024 - 16 - - `fp4_b200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml`` + - `8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml`` * - 8xB200_NVL - Min Latency - 8192 / 1024 - 16 - - `fp4_b200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml`` + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml`` * - B200_NVL - Balanced - 8192 / 1024 - 32 - - `fp4_b200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml`` + - `8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml`` * - 2xB200_NVL - Balanced - 8192 / 1024 - 32 - - `fp4_b200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml`` + - `8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml`` * - 4xB200_NVL - Balanced - 8192 / 1024 - 32 - - `fp4_b200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml`` + - `8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml`` * - 8xB200_NVL - Balanced - 8192 / 1024 - 32 - - `fp4_b200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml`` + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml`` * - B200_NVL - Max Throughput - 8192 / 1024 - 64 - - `fp4_b200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml`` + - `8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml`` * - 2xB200_NVL - Max Throughput - 8192 / 1024 - 64 - - `fp4_b200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml`` + - `8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml`` * - 4xB200_NVL - Max Throughput - 8192 / 1024 - 64 - - `fp4_b200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml`` + - `8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml`` * - 8xB200_NVL - Max Throughput - 8192 / 1024 - 64 - - `fp4_b200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml`` + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 4 - - `fp4_h200_trt_1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml`` + - `1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml`` * - 2xH200_SXM - Min Latency - 1024 / 1024 - 4 - - `fp4_h200_trt_1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml`` + - `1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml`` * - 4xH200_SXM - Min Latency - 1024 / 1024 - 4 - - `fp4_h200_trt_1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml`` + - `1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 1024 - 4 - - `fp4_h200_trt_1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml`` + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 8 - - `fp4_h200_trt_1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml`` + - `1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml`` * - 2xH200_SXM - Min Latency - 1024 / 1024 - 8 - - `fp4_h200_trt_1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml`` + - `1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml`` * - 4xH200_SXM - Min Latency - 1024 / 1024 - 8 - - `fp4_h200_trt_1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml`` + - `1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 1024 - 8 - - `fp4_h200_trt_1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml`` + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 1024 - 16 - - `fp4_h200_trt_1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml`` + - `1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml`` * - 2xH200_SXM - Min Latency - 1024 / 1024 - 16 - - `fp4_h200_trt_1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml`` + - `1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml`` * - 4xH200_SXM - Min Latency - 1024 / 1024 - 16 - - `fp4_h200_trt_1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml`` + - `1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 1024 - 16 - - `fp4_h200_trt_1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml`` + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 1024 / 1024 - 32 - - `fp4_h200_trt_1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml`` + - `1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml`` * - 2xH200_SXM - Balanced - 1024 / 1024 - 32 - - `fp4_h200_trt_1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml`` + - `1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml`` * - 4xH200_SXM - Balanced - 1024 / 1024 - 32 - - `fp4_h200_trt_1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml`` + - `1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml`` * - 8xH200_SXM - Balanced - 1024 / 1024 - 32 - - `fp4_h200_trt_1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml`` + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 1024 / 1024 - 64 - - `fp4_h200_trt_1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml`` + - `1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml`` * - 2xH200_SXM - Max Throughput - 1024 / 1024 - 64 - - `fp4_h200_trt_1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml`` + - `1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml`` * - 4xH200_SXM - Max Throughput - 1024 / 1024 - 64 - - `fp4_h200_trt_1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml`` + - `1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml`` * - 8xH200_SXM - Max Throughput - 1024 / 1024 - 64 - - `fp4_h200_trt_1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml`` + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 4 - - `fp4_h200_trt_1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml`` + - `1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml`` * - 2xH200_SXM - Min Latency - 1024 / 8192 - 4 - - `fp4_h200_trt_1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml`` + - `1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml`` * - 4xH200_SXM - Min Latency - 1024 / 8192 - 4 - - `fp4_h200_trt_1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml`` + - `1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 8192 - 4 - - `fp4_h200_trt_1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml`` + - `1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 8 - - `fp4_h200_trt_1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml`` + - `1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml`` * - 2xH200_SXM - Min Latency - 1024 / 8192 - 8 - - `fp4_h200_trt_1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml`` + - `1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml`` * - 4xH200_SXM - Min Latency - 1024 / 8192 - 8 - - `fp4_h200_trt_1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml`` + - `1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 8192 - 8 - - `fp4_h200_trt_1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml`` + - `1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 1024 / 8192 - 16 - - `fp4_h200_trt_1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml`` + - `1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml`` * - 2xH200_SXM - Min Latency - 1024 / 8192 - 16 - - `fp4_h200_trt_1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml`` + - `1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml`` * - 4xH200_SXM - Min Latency - 1024 / 8192 - 16 - - `fp4_h200_trt_1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml`` + - `1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml`` * - 8xH200_SXM - Min Latency - 1024 / 8192 - 16 - - `fp4_h200_trt_1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml`` + - `1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 1024 / 8192 - 32 - - `fp4_h200_trt_1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml`` + - `1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml`` * - 2xH200_SXM - Balanced - 1024 / 8192 - 32 - - `fp4_h200_trt_1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml`` + - `1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml`` * - 4xH200_SXM - Balanced - 1024 / 8192 - 32 - - `fp4_h200_trt_1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml`` + - `1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml`` * - 8xH200_SXM - Balanced - 1024 / 8192 - 32 - - `fp4_h200_trt_1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml`` + - `1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 1024 / 8192 - 64 - - `fp4_h200_trt_1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml`` + - `1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml`` * - 2xH200_SXM - Max Throughput - 1024 / 8192 - 64 - - `fp4_h200_trt_1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml`` + - `1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml`` * - 4xH200_SXM - Max Throughput - 1024 / 8192 - 64 - - `fp4_h200_trt_1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml`` + - `1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml`` * - 8xH200_SXM - Max Throughput - 1024 / 8192 - 64 - - `fp4_h200_trt_1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml`` + - `1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 4 - - `fp4_h200_trt_8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml`` + - `8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml`` * - 2xH200_SXM - Min Latency - 8192 / 1024 - 4 - - `fp4_h200_trt_8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml`` + - `8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml`` * - 4xH200_SXM - Min Latency - 8192 / 1024 - 4 - - `fp4_h200_trt_8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml`` + - `8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml`` * - 8xH200_SXM - Min Latency - 8192 / 1024 - 4 - - `fp4_h200_trt_8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml`` + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 8 - - `fp4_h200_trt_8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml`` + - `8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml`` * - 2xH200_SXM - Min Latency - 8192 / 1024 - 8 - - `fp4_h200_trt_8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml`` + - `8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml`` * - 4xH200_SXM - Min Latency - 8192 / 1024 - 8 - - `fp4_h200_trt_8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml`` + - `8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml`` * - 8xH200_SXM - Min Latency - 8192 / 1024 - 8 - - `fp4_h200_trt_8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml`` + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml`` * - H200_SXM - Min Latency - 8192 / 1024 - 16 - - `fp4_h200_trt_8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml`` + - `8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml`` * - 2xH200_SXM - Min Latency - 8192 / 1024 - 16 - - `fp4_h200_trt_8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml`` + - `8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml`` * - 4xH200_SXM - Min Latency - 8192 / 1024 - 16 - - `fp4_h200_trt_8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml`` + - `8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml`` * - 8xH200_SXM - Min Latency - 8192 / 1024 - 16 - - `fp4_h200_trt_8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml`` + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml`` * - H200_SXM - Balanced - 8192 / 1024 - 32 - - `fp4_h200_trt_8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml`` + - `8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml`` * - 2xH200_SXM - Balanced - 8192 / 1024 - 32 - - `fp4_h200_trt_8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml`` + - `8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml`` * - 4xH200_SXM - Balanced - 8192 / 1024 - 32 - - `fp4_h200_trt_8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml`` + - `8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml`` * - 8xH200_SXM - Balanced - 8192 / 1024 - 32 - - `fp4_h200_trt_8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml`` + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml`` * - H200_SXM - Max Throughput - 8192 / 1024 - 64 - - `fp4_h200_trt_8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml`` + - `8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml`` * - 2xH200_SXM - Max Throughput - 8192 / 1024 - 64 - - `fp4_h200_trt_8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml`` + - `8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml`` * - 4xH200_SXM - Max Throughput - 8192 / 1024 - 64 - - `fp4_h200_trt_8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml`` + - `8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml`` * - 8xH200_SXM - Max Throughput - 8192 / 1024 - 64 - - `fp4_h200_trt_8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml`` + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml`` .. end-openai/gpt-oss-120b diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index 8386a7c98d1e..dda64d8b0d70 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -47,7 +47,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ /bin/bash ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index 34e097f350cf..016370190ded 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -43,7 +43,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ /bin/bash ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md index 583ef56b4939..d227b2f44091 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md @@ -39,7 +39,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ /bin/bash ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md index 10db2e128f39..509a5cf00f0c 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md @@ -38,7 +38,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ +nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ /bin/bash ``` diff --git a/docs/source/quick-start-guide.md b/docs/source/quick-start-guide.md index 03458cb08fd9..088f70b3ea6c 100644 --- a/docs/source/quick-start-guide.md +++ b/docs/source/quick-start-guide.md @@ -10,7 +10,7 @@ This is the starting point to try out TensorRT LLM. Specifically, this Quick Sta The [TensorRT LLM container](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags) maintained by NVIDIA contains all of the required dependencies pre-installed. You can start the container on a machine with NVIDIA GPUs via: ```bash -docker run --rm -it --ipc host --gpus all --ulimit memlock=-1 --ulimit stack=67108864 -p 8000:8000 nvcr.io/nvidia/tensorrt-llm/release:x.y.z +docker run --rm -it --ipc host --gpus all --ulimit memlock=-1 --ulimit stack=67108864 -p 8000:8000 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 ``` diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 9a069231ca05..a245e3f737b7 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -4,22 +4,19 @@ import yaml -# Mapping for model display names, URLs, and filename shortening prefixes +# Mapping for model display names and URLs MODEL_INFO = { "deepseek-ai/DeepSeek-R1-0528": { "display_name": "DeepSeek-R1", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528", - "prefix_to_strip": "deepseek_r1_0528_", }, "nvidia/DeepSeek-R1-0528-FP4-v2": { "display_name": "DeepSeek-R1 (NVFP4)", "url": "https://huggingface.co/nvidia/DeepSeek-R1-FP4-v2", - "prefix_to_strip": "deepseek_r1_0528_fp4_v2_", }, "openai/gpt-oss-120b": { "display_name": "gpt-oss-120b", "url": "https://huggingface.co/openai/gpt-oss-120b", - "prefix_to_strip": "gpt_oss_120b_", }, } @@ -163,12 +160,6 @@ def generate_rst(yaml_path, output_file=None): config_filename = os.path.basename(full_config_path) - # Shorten config filename if applicable - if model in MODEL_INFO: - prefix = MODEL_INFO[model].get("prefix_to_strip", "") - if config_filename.startswith(prefix): - config_filename = config_filename[len(prefix) :] - github_url = f"https://github.com/NVIDIA/TensorRT-LLM/blob/main/{full_config_path}" config_link = f"`{config_filename} <{github_url}>`_" diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml deleted file mode 100644 index f770a6566e93..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml deleted file mode 100644 index f770a6566e93..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml deleted file mode 100644 index f770a6566e93..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml deleted file mode 100644 index f770a6566e93..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml deleted file mode 100644 index f770a6566e93..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml deleted file mode 100644 index 6660bcea96ae..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml deleted file mode 100644 index 6660bcea96ae..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml deleted file mode 100644 index 6660bcea96ae..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml deleted file mode 100644 index 919a0284099e..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml deleted file mode 100644 index 6660bcea96ae..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 256 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: DEEPGEMM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml deleted file mode 100644 index 008da1df54c4..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml deleted file mode 100644 index 008da1df54c4..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml deleted file mode 100644 index 008da1df54c4..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml deleted file mode 100644 index 008da1df54c4..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml deleted file mode 100644 index 008da1df54c4..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml deleted file mode 100644 index decbb1744a3a..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml deleted file mode 100644 index decbb1744a3a..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml deleted file mode 100644 index decbb1744a3a..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml deleted file mode 100644 index 363eebf52142..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml deleted file mode 100644 index decbb1744a3a..000000000000 --- a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 128 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.75 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml deleted file mode 100644 index c61e3abc15ed..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1216 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml deleted file mode 100644 index fe58a6a32b27..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml deleted file mode 100644 index 2a06d3978d01..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1344 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml deleted file mode 100644 index fe58a6a32b27..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml deleted file mode 100644 index fe58a6a32b27..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml deleted file mode 100644 index fe58a6a32b27..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml deleted file mode 100644 index fe58a6a32b27..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml deleted file mode 100644 index a4a4fe28c7fb..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1216 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml deleted file mode 100644 index 397565e15b50..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml deleted file mode 100644 index 686db04f1fbd..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1344 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml deleted file mode 100644 index 397565e15b50..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml deleted file mode 100644 index 397565e15b50..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml deleted file mode 100644 index 397565e15b50..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml deleted file mode 100644 index 397565e15b50..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 1152 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml deleted file mode 100644 index ace419c0d81e..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8384 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml deleted file mode 100644 index a0f2de5fec79..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml deleted file mode 100644 index 3c812ea3e963..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8512 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml deleted file mode 100644 index a0f2de5fec79..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml deleted file mode 100644 index a0f2de5fec79..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml deleted file mode 100644 index 06f600c1cd90..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml deleted file mode 100644 index a0f2de5fec79..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml deleted file mode 100644 index 5334ed3cf545..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8384 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml deleted file mode 100644 index 382a3c9045da..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml deleted file mode 100644 index 639fdde94a18..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8512 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml deleted file mode 100644 index 382a3c9045da..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml deleted file mode 100644 index 382a3c9045da..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml deleted file mode 100644 index 930a6253084f..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: true -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: CUTLASS -attention_dp_config: - batching_wait_iters: 0 - enable_balance: true - timeout_iters: 60 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml deleted file mode 100644 index 382a3c9045da..000000000000 --- a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ /dev/null @@ -1,18 +0,0 @@ -cuda_graph_config: - enable_padding: true - max_batch_size: 512 -enable_attention_dp: false -print_iter_log: true -kv_cache_config: - dtype: fp8 - free_gpu_memory_fraction: 0.8 - enable_block_reuse: false -stream_interval: 10 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 8320 -max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml deleted file mode 100644 index 1d4df970107d..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml deleted file mode 100644 index 7d65f54710bf..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml deleted file mode 100644 index ca850a775852..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml deleted file mode 100644 index 345b0e501388..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml deleted file mode 100644 index 5fa5e373d2e5..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml deleted file mode 100644 index 7b392ada8d6b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml deleted file mode 100644 index e8212dd139ff..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml deleted file mode 100644 index ab22a7baf682..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml deleted file mode 100644 index 3f8265048033..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml deleted file mode 100644 index b07960f33ddd..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml deleted file mode 100644 index e078ea3d6d32..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml deleted file mode 100644 index 15f5a3ca501d..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml deleted file mode 100644 index cdbb40a3eb47..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml deleted file mode 100644 index c5854b6dafb1..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml deleted file mode 100644 index 0ac44311750b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml deleted file mode 100644 index a18faa262246..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml deleted file mode 100644 index 4ce42b3ce83f..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml deleted file mode 100644 index 966138c163c4..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml deleted file mode 100644 index a322f0681d28..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml deleted file mode 100644 index 644d2dabb489..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml deleted file mode 100644 index 31544aa9f4f6..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml deleted file mode 100644 index ec0ea7b2ba31..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml deleted file mode 100644 index 249b14723ff3..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml deleted file mode 100644 index 21de3414a842..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml deleted file mode 100644 index 315b1add429c..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml deleted file mode 100644 index 56e1b648bd78..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml deleted file mode 100644 index 4e02fe671b4e..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml deleted file mode 100644 index 4bc360839a97..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml deleted file mode 100644 index 584fb5ae1aee..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml deleted file mode 100644 index 6ab46126d56b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml deleted file mode 100644 index ef539d3befa9..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml deleted file mode 100644 index 40dc75208448..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml deleted file mode 100644 index 3e0f48e7e1bf..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml deleted file mode 100644 index 2e3721c712dc..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml deleted file mode 100644 index 098e7ec388cf..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml deleted file mode 100644 index 45d77f70bd23..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml deleted file mode 100644 index 9436b079590a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml deleted file mode 100644 index a2917bfd5b19..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml deleted file mode 100644 index 702d3bc00cb1..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml deleted file mode 100644 index c0b90314c3d0..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml deleted file mode 100644 index 31544aa9f4f6..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml deleted file mode 100644 index ec0ea7b2ba31..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml deleted file mode 100644 index 249b14723ff3..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml deleted file mode 100644 index 21de3414a842..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml deleted file mode 100644 index 315b1add429c..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml deleted file mode 100644 index 56e1b648bd78..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml deleted file mode 100644 index 4e02fe671b4e..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml deleted file mode 100644 index 4bc360839a97..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml deleted file mode 100644 index 584fb5ae1aee..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml deleted file mode 100644 index 6ab46126d56b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml deleted file mode 100644 index ef539d3befa9..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml deleted file mode 100644 index 40dc75208448..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml deleted file mode 100644 index 3e0f48e7e1bf..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml deleted file mode 100644 index 2e3721c712dc..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml deleted file mode 100644 index 098e7ec388cf..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml deleted file mode 100644 index 45d77f70bd23..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml deleted file mode 100644 index 9436b079590a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml deleted file mode 100644 index a2917bfd5b19..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml deleted file mode 100644 index 702d3bc00cb1..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml deleted file mode 100644 index c0b90314c3d0..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml +++ /dev/null @@ -1,22 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 - NCCL_GRAPH_REGISTER: 0 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: fp8 - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -print_iter_log: true -stream_interval: 20 -num_postprocess_workers: 4 -moe_config: - backend: TRTLLM -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml deleted file mode 100644 index 2eea897e2f58..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml deleted file mode 100644 index 1a0d44fb2778..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml deleted file mode 100644 index 82662456f032..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml deleted file mode 100644 index 57d8e2ada29b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml deleted file mode 100644 index 87e34788d7c6..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml deleted file mode 100644 index 57b4b87fc72d..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml deleted file mode 100644 index 0d796e475135..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml deleted file mode 100644 index f6c41d8bbdf7..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml deleted file mode 100644 index fdec025db8b9..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml deleted file mode 100644 index 8565e82e3623..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml deleted file mode 100644 index 4773067517e2..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml deleted file mode 100644 index 5e0d27c5ead3..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml deleted file mode 100644 index 9b135c0a3215..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml deleted file mode 100644 index 6874784b9f29..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml deleted file mode 100644 index cc1d2d8ac986..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml deleted file mode 100644 index f7e46b17a369..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml deleted file mode 100644 index 1b1b874c3e1b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml deleted file mode 100644 index 28a7f3d17c69..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml deleted file mode 100644 index 8036e7439962..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml deleted file mode 100644 index 12289904eda8..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml deleted file mode 100644 index 7ccdc4ae1146..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml deleted file mode 100644 index ea6a93ba64bb..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml deleted file mode 100644 index a0149f2ab5c5..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml deleted file mode 100644 index 3ae56a300ad6..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml deleted file mode 100644 index c18bc3c7581c..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml deleted file mode 100644 index e88b4e05fe8a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml deleted file mode 100644 index 95b8e207330d..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml deleted file mode 100644 index c35b691a81c5..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml deleted file mode 100644 index ce0f7c27577c..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml deleted file mode 100644 index 344166bc325b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml deleted file mode 100644 index 4f895199b1f2..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml deleted file mode 100644 index ca549de3d28a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml deleted file mode 100644 index b87044bbc0ea..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml deleted file mode 100644 index 9af104970e78..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml deleted file mode 100644 index 7440c3fcb7e7..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml deleted file mode 100644 index b1d8a6eead07..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml deleted file mode 100644 index f8c7fec13a7a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml deleted file mode 100644 index f9cb8feb6956..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml deleted file mode 100644 index a9124d700757..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml deleted file mode 100644 index 7c2507ace7aa..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml deleted file mode 100644 index 7ccdc4ae1146..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml deleted file mode 100644 index ea6a93ba64bb..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml deleted file mode 100644 index a0149f2ab5c5..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml deleted file mode 100644 index 3ae56a300ad6..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml deleted file mode 100644 index c18bc3c7581c..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 1 -moe_expert_parallel_size: 1 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml deleted file mode 100644 index e88b4e05fe8a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml deleted file mode 100644 index 95b8e207330d..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml deleted file mode 100644 index c35b691a81c5..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml deleted file mode 100644 index ce0f7c27577c..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml deleted file mode 100644 index 344166bc325b..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 2 -moe_expert_parallel_size: 2 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml deleted file mode 100644 index 4f895199b1f2..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml deleted file mode 100644 index ca549de3d28a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml deleted file mode 100644 index b87044bbc0ea..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml deleted file mode 100644 index 9af104970e78..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml deleted file mode 100644 index 7440c3fcb7e7..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 4 -moe_expert_parallel_size: 4 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml deleted file mode 100644 index b1d8a6eead07..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 16 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml deleted file mode 100644 index f8c7fec13a7a..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 32 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml deleted file mode 100644 index f9cb8feb6956..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 4 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml deleted file mode 100644 index a9124d700757..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 64 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml deleted file mode 100644 index 7c2507ace7aa..000000000000 --- a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml +++ /dev/null @@ -1,21 +0,0 @@ -env_overrides: - TRTLLM_ENABLE_PDL: 1 -cuda_graph_config: - enable_padding: true - max_batch_size: 8 -enable_attention_dp: false -kv_cache_config: - dtype: auto - enable_block_reuse: false - free_gpu_memory_fraction: 0.85 -moe_config: - backend: TRITON -num_postprocess_workers: 4 -print_iter_log: true -stream_interval: 20 -tensor_parallel_size: 8 -moe_expert_parallel_size: 8 -trust_remote_code: true -backend: pytorch -max_num_tokens: 20000 -max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/scenario_list.yaml b/tensorrt_llm/configure/database/scenario_list.yaml index c918759403f1..031b7846bca2 100644 --- a/tensorrt_llm/configure/database/scenario_list.yaml +++ b/tensorrt_llm/configure/database/scenario_list.yaml @@ -3,1174 +3,1174 @@ isl: 1024 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc128.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc16.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc256.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc32.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc4.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc64.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp4_conc8.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc128.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc16.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc256.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc32.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc4.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc64.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_1k1k_tp8_conc8.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc128.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc16.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc256.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc32.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc4.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc64.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp4_conc8.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc128.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc16.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc256.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc32.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc4.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc64.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/deepseek_r1_0528_fp4_v2_fp4_b200_trt_8k1k_tp8_conc8.yaml + config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc16.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc32.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc4.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc64.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_1k1k_tp8_conc8.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc16.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc32.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc4.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc64.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/deepseek_r1_0528_fp8_b200_trt_8k1k_tp8_conc8.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc16.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc32.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc4.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc64.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_1k1k_tp8_conc8.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc16.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc32.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc4.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc64.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/deepseek_r1_0528_fp8_h200_trt_8k1k_tp8_conc8.yaml + config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp1_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp2_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp4_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k1k_tp8_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp1_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp2_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp4_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_1k8k_tp8_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp1_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp2_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp4_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc16.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc32.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc4.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc64.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/gpt_oss_120b_fp4_b200_trt_8k1k_tp8_conc8.yaml + config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp1_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp2_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp4_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k1k_tp8_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp1_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp2_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp4_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_1k8k_tp8_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp1_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp2_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp4_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc16.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc32.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc4.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc64.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/gpt_oss_120b_fp4_h200_trt_8k1k_tp8_conc8.yaml + config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml num_gpus: 8 From 65c7cd65294afb7b81d61e5b7f3d569787136aaa Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Sun, 7 Dec 2025 23:30:20 -0800 Subject: [PATCH 27/39] revert x.y.z version changes Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../commands/trtllm-serve/run-benchmark-with-trtllm-serve.md | 2 +- .../deployment-guide-for-deepseek-r1-on-trtllm.md | 2 +- .../deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md | 2 +- .../deployment-guide-for-llama3.3-70b-on-trtllm.md | 2 +- .../deployment-guide-for-llama4-scout-on-trtllm.md | 2 +- docs/source/quick-start-guide.md | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md b/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md index 81c296fe9db3..04e188ec0f8d 100644 --- a/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md +++ b/docs/source/commands/trtllm-serve/run-benchmark-with-trtllm-serve.md @@ -25,7 +25,7 @@ TensorRT LLM distributes the pre-built container on [NGC Catalog](https://catalo You can launch the container using the following command: ```bash -docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 +docker run --rm -it --ipc host -p 8000:8000 --gpus all --ulimit memlock=-1 --ulimit stack=67108864 nvcr.io/nvidia/tensorrt-llm/release:x.y.z ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index dda64d8b0d70..8386a7c98d1e 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -47,7 +47,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ +nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ /bin/bash ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index 016370190ded..34e097f350cf 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -43,7 +43,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ +nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ /bin/bash ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md index d227b2f44091..583ef56b4939 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md @@ -39,7 +39,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ +nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ /bin/bash ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md index 509a5cf00f0c..10db2e128f39 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md @@ -38,7 +38,7 @@ docker run --rm -it \ -p 8000:8000 \ -v ~/.cache:/root/.cache:rw \ --name tensorrt_llm \ -nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 \ +nvcr.io/nvidia/tensorrt-llm/release:x.y.z \ /bin/bash ``` diff --git a/docs/source/quick-start-guide.md b/docs/source/quick-start-guide.md index 088f70b3ea6c..03458cb08fd9 100644 --- a/docs/source/quick-start-guide.md +++ b/docs/source/quick-start-guide.md @@ -10,7 +10,7 @@ This is the starting point to try out TensorRT LLM. Specifically, this Quick Sta The [TensorRT LLM container](https://catalog.ngc.nvidia.com/orgs/nvidia/teams/tensorrt-llm/containers/release/tags) maintained by NVIDIA contains all of the required dependencies pre-installed. You can start the container on a machine with NVIDIA GPUs via: ```bash -docker run --rm -it --ipc host --gpus all --ulimit memlock=-1 --ulimit stack=67108864 -p 8000:8000 nvcr.io/nvidia/tensorrt-llm/release:1.2.0rc5 +docker run --rm -it --ipc host --gpus all --ulimit memlock=-1 --ulimit stack=67108864 -p 8000:8000 nvcr.io/nvidia/tensorrt-llm/release:x.y.z ``` From 6a0a11d4128d97bdb7df848f3b400db9bbcbef50 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Sun, 7 Dec 2025 23:31:05 -0800 Subject: [PATCH 28/39] add the untracked yaml files Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../B200/1k1k_tp8_conc16.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc32.yaml | 18 +++++++++++++++ .../DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc64.yaml | 18 +++++++++++++++ .../DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml | 18 +++++++++++++++ .../B200/8k1k_tp8_conc16.yaml | 18 +++++++++++++++ .../B200/8k1k_tp8_conc32.yaml | 18 +++++++++++++++ .../DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml | 18 +++++++++++++++ .../B200/8k1k_tp8_conc64.yaml | 22 +++++++++++++++++++ .../DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml | 18 +++++++++++++++ .../H200/1k1k_tp8_conc16.yaml | 18 +++++++++++++++ .../H200/1k1k_tp8_conc32.yaml | 18 +++++++++++++++ .../DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml | 18 +++++++++++++++ .../H200/1k1k_tp8_conc64.yaml | 18 +++++++++++++++ .../DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml | 18 +++++++++++++++ .../H200/8k1k_tp8_conc16.yaml | 18 +++++++++++++++ .../H200/8k1k_tp8_conc32.yaml | 18 +++++++++++++++ .../DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml | 18 +++++++++++++++ .../H200/8k1k_tp8_conc64.yaml | 22 +++++++++++++++++++ .../DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml | 18 +++++++++++++++ .../B200/1k1k_tp4_conc128.yaml | 18 +++++++++++++++ .../B200/1k1k_tp4_conc16.yaml | 18 +++++++++++++++ .../B200/1k1k_tp4_conc256.yaml | 22 +++++++++++++++++++ .../B200/1k1k_tp4_conc32.yaml | 18 +++++++++++++++ .../B200/1k1k_tp4_conc4.yaml | 18 +++++++++++++++ .../B200/1k1k_tp4_conc64.yaml | 18 +++++++++++++++ .../B200/1k1k_tp4_conc8.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc128.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc16.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc256.yaml | 22 +++++++++++++++++++ .../B200/1k1k_tp8_conc32.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc4.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc64.yaml | 18 +++++++++++++++ .../B200/1k1k_tp8_conc8.yaml | 18 +++++++++++++++ .../B200/8k1k_tp4_conc128.yaml | 22 +++++++++++++++++++ .../B200/8k1k_tp4_conc16.yaml | 18 +++++++++++++++ .../B200/8k1k_tp4_conc256.yaml | 22 +++++++++++++++++++ .../B200/8k1k_tp4_conc32.yaml | 18 +++++++++++++++ .../B200/8k1k_tp4_conc4.yaml | 18 +++++++++++++++ .../B200/8k1k_tp4_conc64.yaml | 22 +++++++++++++++++++ .../B200/8k1k_tp4_conc8.yaml | 18 +++++++++++++++ .../B200/8k1k_tp8_conc128.yaml | 22 +++++++++++++++++++ .../B200/8k1k_tp8_conc16.yaml | 18 +++++++++++++++ .../B200/8k1k_tp8_conc256.yaml | 22 +++++++++++++++++++ .../B200/8k1k_tp8_conc32.yaml | 18 +++++++++++++++ .../B200/8k1k_tp8_conc4.yaml | 18 +++++++++++++++ .../B200/8k1k_tp8_conc64.yaml | 22 +++++++++++++++++++ .../B200/8k1k_tp8_conc8.yaml | 18 +++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp1_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp1_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp1_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp1_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp1_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp2_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp2_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp2_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp2_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp2_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp4_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp4_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp4_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp4_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp4_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp8_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp8_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp8_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp8_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k1k_tp8_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp1_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp1_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp1_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp1_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp1_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp2_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp2_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp2_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp2_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp2_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp4_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp4_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp4_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp4_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp4_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp8_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp8_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp8_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp8_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/1k8k_tp8_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp1_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp1_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp1_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp1_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp1_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp2_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp2_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp2_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp2_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp2_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp4_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp4_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp4_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp4_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp4_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp8_conc16.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp8_conc32.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp8_conc4.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp8_conc64.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/B200/8k1k_tp8_conc8.yaml | 22 +++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp1_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp1_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp1_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp1_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp1_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp2_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp2_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp2_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp2_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp2_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp4_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp4_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp4_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp4_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp4_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp8_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp8_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp8_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp8_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k1k_tp8_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp1_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp1_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp1_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp1_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp1_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp2_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp2_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp2_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp2_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp2_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp4_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp4_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp4_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp4_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp4_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp8_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp8_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp8_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp8_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/1k8k_tp8_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp1_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp1_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp1_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp1_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp1_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp2_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp2_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp2_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp2_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp2_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp4_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp4_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp4_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp4_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp4_conc8.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp8_conc16.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp8_conc32.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp8_conc4.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp8_conc64.yaml | 21 ++++++++++++++++++ .../gpt-oss-120b/H200/8k1k_tp8_conc8.yaml | 21 ++++++++++++++++++ 168 files changed, 3484 insertions(+) create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml create mode 100644 tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..f770a6566e93 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..f770a6566e93 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..f770a6566e93 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..f770a6566e93 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..f770a6566e93 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..6660bcea96ae --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..6660bcea96ae --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..6660bcea96ae --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..919a0284099e --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..6660bcea96ae --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 256 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: DEEPGEMM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..008da1df54c4 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..008da1df54c4 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..008da1df54c4 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..008da1df54c4 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..008da1df54c4 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..decbb1744a3a --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..decbb1744a3a --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..decbb1744a3a --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..363eebf52142 --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..decbb1744a3a --- /dev/null +++ b/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 128 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.75 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml new file mode 100644 index 000000000000..c61e3abc15ed --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1216 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..fe58a6a32b27 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..2a06d3978d01 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1344 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..fe58a6a32b27 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..fe58a6a32b27 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..fe58a6a32b27 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..fe58a6a32b27 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml new file mode 100644 index 000000000000..a4a4fe28c7fb --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1216 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..397565e15b50 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml new file mode 100644 index 000000000000..686db04f1fbd --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1344 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..397565e15b50 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..397565e15b50 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..397565e15b50 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..397565e15b50 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 1152 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml new file mode 100644 index 000000000000..ace419c0d81e --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8384 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..a0f2de5fec79 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml new file mode 100644 index 000000000000..3c812ea3e963 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8512 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..a0f2de5fec79 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..a0f2de5fec79 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..06f600c1cd90 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..a0f2de5fec79 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml new file mode 100644 index 000000000000..5334ed3cf545 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8384 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..382a3c9045da --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml new file mode 100644 index 000000000000..639fdde94a18 --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8512 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..382a3c9045da --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..382a3c9045da --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..930a6253084f --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: true +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: CUTLASS +attention_dp_config: + batching_wait_iters: 0 + enable_balance: true + timeout_iters: 60 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..382a3c9045da --- /dev/null +++ b/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml @@ -0,0 +1,18 @@ +cuda_graph_config: + enable_padding: true + max_batch_size: 512 +enable_attention_dp: false +print_iter_log: true +kv_cache_config: + dtype: fp8 + free_gpu_memory_fraction: 0.8 + enable_block_reuse: false +stream_interval: 10 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 8320 +max_seq_len: 9416 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..1d4df970107d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..7d65f54710bf --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..ca850a775852 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..345b0e501388 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..5fa5e373d2e5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..7b392ada8d6b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..e8212dd139ff --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..ab22a7baf682 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..3f8265048033 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..b07960f33ddd --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..e078ea3d6d32 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..15f5a3ca501d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..cdbb40a3eb47 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..c5854b6dafb1 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..0ac44311750b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..a18faa262246 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..4ce42b3ce83f --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..966138c163c4 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..a322f0681d28 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..644d2dabb489 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml new file mode 100644 index 000000000000..31544aa9f4f6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml new file mode 100644 index 000000000000..ec0ea7b2ba31 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml new file mode 100644 index 000000000000..249b14723ff3 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml new file mode 100644 index 000000000000..21de3414a842 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml new file mode 100644 index 000000000000..315b1add429c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml new file mode 100644 index 000000000000..56e1b648bd78 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml new file mode 100644 index 000000000000..4e02fe671b4e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml new file mode 100644 index 000000000000..4bc360839a97 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml new file mode 100644 index 000000000000..584fb5ae1aee --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml new file mode 100644 index 000000000000..6ab46126d56b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml new file mode 100644 index 000000000000..ef539d3befa9 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml new file mode 100644 index 000000000000..40dc75208448 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml new file mode 100644 index 000000000000..3e0f48e7e1bf --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml new file mode 100644 index 000000000000..2e3721c712dc --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml new file mode 100644 index 000000000000..098e7ec388cf --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml new file mode 100644 index 000000000000..45d77f70bd23 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml new file mode 100644 index 000000000000..9436b079590a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml new file mode 100644 index 000000000000..a2917bfd5b19 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml new file mode 100644 index 000000000000..702d3bc00cb1 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml new file mode 100644 index 000000000000..c0b90314c3d0 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..31544aa9f4f6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..ec0ea7b2ba31 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..249b14723ff3 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..21de3414a842 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..315b1add429c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..56e1b648bd78 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..4e02fe671b4e --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..4bc360839a97 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..584fb5ae1aee --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..6ab46126d56b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..ef539d3befa9 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..40dc75208448 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..3e0f48e7e1bf --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..2e3721c712dc --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..098e7ec388cf --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..45d77f70bd23 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..9436b079590a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..a2917bfd5b19 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..702d3bc00cb1 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..c0b90314c3d0 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml @@ -0,0 +1,22 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 + NCCL_GRAPH_REGISTER: 0 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: fp8 + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +print_iter_log: true +stream_interval: 20 +num_postprocess_workers: 4 +moe_config: + backend: TRTLLM +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..2eea897e2f58 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..1a0d44fb2778 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..82662456f032 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..57d8e2ada29b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..87e34788d7c6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..57b4b87fc72d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..0d796e475135 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..f6c41d8bbdf7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..fdec025db8b9 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..8565e82e3623 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..4773067517e2 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..5e0d27c5ead3 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..9b135c0a3215 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..6874784b9f29 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..cc1d2d8ac986 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..f7e46b17a369 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..1b1b874c3e1b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..28a7f3d17c69 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..8036e7439962 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..12289904eda8 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 2068 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml new file mode 100644 index 000000000000..7ccdc4ae1146 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml new file mode 100644 index 000000000000..ea6a93ba64bb --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml new file mode 100644 index 000000000000..a0149f2ab5c5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml new file mode 100644 index 000000000000..3ae56a300ad6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml new file mode 100644 index 000000000000..c18bc3c7581c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml new file mode 100644 index 000000000000..e88b4e05fe8a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml new file mode 100644 index 000000000000..95b8e207330d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml new file mode 100644 index 000000000000..c35b691a81c5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml new file mode 100644 index 000000000000..ce0f7c27577c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml new file mode 100644 index 000000000000..344166bc325b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml new file mode 100644 index 000000000000..4f895199b1f2 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml new file mode 100644 index 000000000000..ca549de3d28a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml new file mode 100644 index 000000000000..b87044bbc0ea --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml new file mode 100644 index 000000000000..9af104970e78 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml new file mode 100644 index 000000000000..7440c3fcb7e7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml new file mode 100644 index 000000000000..b1d8a6eead07 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml new file mode 100644 index 000000000000..f8c7fec13a7a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml new file mode 100644 index 000000000000..f9cb8feb6956 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml new file mode 100644 index 000000000000..a9124d700757 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml new file mode 100644 index 000000000000..7c2507ace7aa --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml new file mode 100644 index 000000000000..7ccdc4ae1146 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml new file mode 100644 index 000000000000..ea6a93ba64bb --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml new file mode 100644 index 000000000000..a0149f2ab5c5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml new file mode 100644 index 000000000000..3ae56a300ad6 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml new file mode 100644 index 000000000000..c18bc3c7581c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 1 +moe_expert_parallel_size: 1 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml new file mode 100644 index 000000000000..e88b4e05fe8a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml new file mode 100644 index 000000000000..95b8e207330d --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml new file mode 100644 index 000000000000..c35b691a81c5 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml new file mode 100644 index 000000000000..ce0f7c27577c --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml new file mode 100644 index 000000000000..344166bc325b --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 2 +moe_expert_parallel_size: 2 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml new file mode 100644 index 000000000000..4f895199b1f2 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml new file mode 100644 index 000000000000..ca549de3d28a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml new file mode 100644 index 000000000000..b87044bbc0ea --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml new file mode 100644 index 000000000000..9af104970e78 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml new file mode 100644 index 000000000000..7440c3fcb7e7 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 4 +moe_expert_parallel_size: 4 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml new file mode 100644 index 000000000000..b1d8a6eead07 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 16 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml new file mode 100644 index 000000000000..f8c7fec13a7a --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 32 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml new file mode 100644 index 000000000000..f9cb8feb6956 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 4 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml new file mode 100644 index 000000000000..a9124d700757 --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 64 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml new file mode 100644 index 000000000000..7c2507ace7aa --- /dev/null +++ b/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml @@ -0,0 +1,21 @@ +env_overrides: + TRTLLM_ENABLE_PDL: 1 +cuda_graph_config: + enable_padding: true + max_batch_size: 8 +enable_attention_dp: false +kv_cache_config: + dtype: auto + enable_block_reuse: false + free_gpu_memory_fraction: 0.85 +moe_config: + backend: TRITON +num_postprocess_workers: 4 +print_iter_log: true +stream_interval: 20 +tensor_parallel_size: 8 +moe_expert_parallel_size: 8 +trust_remote_code: true +backend: pytorch +max_num_tokens: 20000 +max_seq_len: 9236 From f36d3a701f3e7ddd64566f620698a862582fb674 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Sun, 7 Dec 2025 23:32:58 -0800 Subject: [PATCH 29/39] add copyright headers Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- scripts/generate_config_table.py | 16 ++++++++++++++++ tensorrt_llm/configure/database.py | 16 ++++++++++++++++ .../unittest/tools/test_generate_config_table.py | 15 +++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index a245e3f737b7..e5d1fe8b410b 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -1,3 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + import os import sys from collections import defaultdict diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 4aea70bf7dac..9faac9495f7a 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -1,3 +1,19 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + from pathlib import Path from typing import Any, Dict, Iterator, List diff --git a/tests/unittest/tools/test_generate_config_table.py b/tests/unittest/tools/test_generate_config_table.py index a19364641b3e..898169496056 100644 --- a/tests/unittest/tools/test_generate_config_table.py +++ b/tests/unittest/tools/test_generate_config_table.py @@ -1,3 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + import os import sys import tempfile From b1dd58b51e1de66285e58821ee8d000e04e8e00b Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 00:04:52 -0800 Subject: [PATCH 30/39] correct the deepseek http url Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- docs/source/deployment-guide/comprehensive_table.rst | 4 ++-- scripts/generate_config_table.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 38a7476300f6..c658c914ee65 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -160,8 +160,8 @@ .. _nvidia/DeepSeek-R1-0528-FP4-v2: -`DeepSeek-R1 (NVFP4) `_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +`DeepSeek-R1 (NVFP4) `_ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. list-table:: :width: 100% diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index e5d1fe8b410b..cef8dde5371e 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -28,7 +28,7 @@ }, "nvidia/DeepSeek-R1-0528-FP4-v2": { "display_name": "DeepSeek-R1 (NVFP4)", - "url": "https://huggingface.co/nvidia/DeepSeek-R1-FP4-v2", + "url": "https://huggingface.co/nvidia/DeepSeek-R1-0528-FP4-v2", }, "openai/gpt-oss-120b": { "display_name": "gpt-oss-120b", From fea5dc5bd9bdef61a391bd91515f2ea444e9bd31 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 01:18:30 -0800 Subject: [PATCH 31/39] trim database script, include into table script Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- scripts/generate_config_table.py | 79 ++++++++++-------------------- tensorrt_llm/configure/database.py | 50 +++---------------- 2 files changed, 33 insertions(+), 96 deletions(-) diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index cef8dde5371e..14ad1f5fbff9 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -17,10 +17,14 @@ import os import sys from collections import defaultdict +from pathlib import Path -import yaml +SCRIPT_DIR = Path(__file__).parent.resolve() +REPO_ROOT = SCRIPT_DIR.parent +sys.path.insert(0, str(REPO_ROOT)) + +from tensorrt_llm.configure.database import DATABASE_LIST_PATH, RecipeList # noqa: E402 -# Mapping for model display names and URLs MODEL_INFO = { "deepseek-ai/DeepSeek-R1-0528": { "display_name": "DeepSeek-R1", @@ -41,24 +45,18 @@ def generate_rst(yaml_path, output_file=None): """Generate RST table from YAML config database. Args: - yaml_path: Path to scenario_list.yaml + yaml_path: Path to scenario_list.yaml (str or Path) output_file: Optional output file path. If None, prints to stdout. """ - with open(yaml_path, "r") as f: - data = yaml.safe_load(f) + recipe_list = RecipeList.from_yaml(Path(yaml_path)) - # Group by model first, then by key attributes - # Structure: model -> (gpu, isl, osl) -> list of entries + # Group by model -> (gpu, isl, osl) -> list of recipes model_groups = defaultdict(lambda: defaultdict(list)) - for entry in data: - model = entry.get("model", "Unknown Model") - key = (entry.get("gpu"), entry.get("isl"), entry.get("osl")) - model_groups[model][key].append(entry) + for recipe in recipe_list: + key = (recipe.gpu, recipe.isl, recipe.osl) + model_groups[recipe.model][key].append(recipe) - # Prepare output lines lines = [] - - # Add introductory note about ISL/OSL constraints lines.append(".. note::") lines.append("") lines.append( @@ -89,32 +87,27 @@ def generate_rst(yaml_path, output_file=None): ) lines.append("") - # Sort models alphabetically sorted_models = sorted(model_groups.keys()) for model in sorted_models: lines.append(f".. start-{model}") lines.append("") - # Determine title text if model in MODEL_INFO: info = MODEL_INFO[model] title_text = f"`{info['display_name']} <{info['url']}>`_" else: title_text = model - # Section Header for Model lines.append(f".. _{model}:") lines.append("") lines.append(title_text) lines.append("^" * len(title_text)) lines.append("") - # Table Header lines.append(".. list-table::") lines.append(" :width: 100%") lines.append(" :header-rows: 1") - # Widths: GPU, Perf Profile, ISL/OSL, Concurrency, Config, Command lines.append(" :widths: 12 15 15 13 20 25") lines.append("") lines.append(" * - GPU") @@ -124,36 +117,29 @@ def generate_rst(yaml_path, output_file=None): lines.append(" - Config") lines.append(" - Command") - # Process entries for this model subgroups = model_groups[model] - - # Sort subgroups by GPU, ISL, OSL sorted_keys = sorted( subgroups.keys(), key=lambda k: (str(k[0]), int(k[1] or 0), int(k[2] or 0)) ) for key in sorted_keys: entries = subgroups[key] - # Sort by concurrency - entries.sort(key=lambda x: int(x.get("concurrency", 0))) + entries.sort(key=lambda x: x.concurrency) - # Get concurrency range for this group to determine profile - min_conc = int(entries[0].get("concurrency", 0)) - max_conc = int(entries[-1].get("concurrency", 0)) + min_conc = entries[0].concurrency + max_conc = entries[-1].concurrency conc_range = max_conc - min_conc for entry in entries: - gpu = entry.get("gpu", "N/A") - num_gpus = entry.get("num_gpus", 1) + gpu = entry.gpu + num_gpus = entry.num_gpus gpu_display = f"{num_gpus}x{gpu}" if num_gpus and num_gpus > 1 else gpu - isl = entry.get("isl", "N/A") - osl = entry.get("osl", "N/A") - conc = int(entry.get("concurrency", 0)) - config_path = entry.get("config_path", "") + isl = entry.isl + osl = entry.osl + conc = entry.concurrency + config_path = entry.config_path - # Determine profile based on relative position in concurrency range if len(entries) == 1: - # Single entry: use concurrency value as heuristic if conc <= 16: profile = "Min Latency" elif conc >= 64: @@ -161,7 +147,6 @@ def generate_rst(yaml_path, output_file=None): else: profile = "Balanced" else: - # Multiple entries: use relative position relative_pos = (conc - min_conc) / conc_range if conc_range > 0 else 0.5 if relative_pos < 0.33: @@ -186,11 +171,10 @@ def generate_rst(yaml_path, output_file=None): lines.append(f" - {config_link}") lines.append(f" - ``{command}``") - lines.append("") # Space between tables + lines.append("") lines.append(f".. end-{model}") lines.append("") - # Output to file or stdout output_text = "\n".join(lines) if output_file: with open(output_file, "w") as f: @@ -201,22 +185,9 @@ def generate_rst(yaml_path, output_file=None): if __name__ == "__main__": - # Assume script is run from repo root or tools dir - script_dir = os.path.dirname(os.path.abspath(__file__)) - # script is in tools/, repo root is parent - repo_root = os.path.dirname(script_dir) - - # Just to be safe, look for the file in fixed location relative to this script - yaml_path = os.path.join(repo_root, "tensorrt_llm/configure/database/scenario_list.yaml") - - if not os.path.exists(yaml_path): - # Try relative to CWD if script logic above fails (e.g. symlinks or strange structure) - yaml_path = "tensorrt_llm/configure/database/scenario_list.yaml" - - if not os.path.exists(yaml_path): + yaml_path = DATABASE_LIST_PATH + if not yaml_path.exists(): print(f"Error: YAML file not found at {yaml_path}", file=sys.stderr) sys.exit(1) - - # Generate to separate file - output_path = os.path.join(repo_root, "docs/source/deployment-guide/comprehensive_table.rst") + output_path = REPO_ROOT / "docs/source/deployment-guide/comprehensive_table.rst" generate_rst(yaml_path, output_file=output_path) diff --git a/tensorrt_llm/configure/database.py b/tensorrt_llm/configure/database.py index 9faac9495f7a..a2ef3fba6867 100644 --- a/tensorrt_llm/configure/database.py +++ b/tensorrt_llm/configure/database.py @@ -17,7 +17,6 @@ from pathlib import Path from typing import Any, Dict, Iterator, List -import pandas as pd import yaml from pydantic import BaseModel, Field, RootModel @@ -35,18 +34,11 @@ class RecipeConstraints(BaseModel): config_path: str = Field(description="Configuration path") num_gpus: int = Field(description="Number of GPUs") - def to_pandas_row(self) -> pd.Series: - """Convert recipe record to pandas Series. - - Returns: - pd.Series: Pandas Series containing the recipe record. - """ - return pd.Series(self.model_dump()) - def load_config(self) -> Dict[str, Any]: - """Load configuration from the configuration path.""" - with open(self.config_path, "r") as f: - return Recipe(**yaml.safe_load(f, Loader=yaml.FullLoader)) + """Load and return the YAML config at config_path.""" + with open(self.config_path) as f: + data = yaml.safe_load(f) + return data if data is not None else {} class Recipe(BaseModel): @@ -60,39 +52,13 @@ class Recipe(BaseModel): class RecipeList(RootModel[List[RecipeConstraints]]): @classmethod def from_yaml(cls, yaml_path: Path) -> "RecipeList": - """Load recipe list from YAML file and validate the data. - - Args: - yaml_path (Path): Path to the YAML file containing the recipe list. - - Returns: - RecipeList: Recipe list object. - """ - with open(yaml_path, "r") as f: - data = yaml.safe_load(f, Loader=yaml.FullLoader) + """Load and validate recipe list from YAML file.""" + with open(yaml_path) as f: + data = yaml.safe_load(f) return cls(data) - def to_pandas_dataframe(self) -> pd.DataFrame: - """Convert recipe list to pandas DataFrame. - - Returns: - pd.DataFrame: Pandas DataFrame where each row contains a recipe record. - """ - return pd.DataFrame([record.to_pandas_row() for record in self.root]) - def __iter__(self) -> Iterator[RecipeConstraints]: - """Iterate over the recipe list. - - Returns: - Iterator[RecipeRecord]: Iterator over the recipe list. - """ - for record in self.root: - yield record + return iter(self.root) def __len__(self) -> int: - """Get the number of recipes in the list. - - Returns: - int: Number of recipes in the list. - """ return len(self.root) From e71d82725ff62b39cea38cf6a117bb63fa4e7d28 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 01:26:27 -0800 Subject: [PATCH 32/39] split min/max and high/low throughput/latency Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide/comprehensive_table.rst | 208 +++++++++--------- scripts/generate_config_table.py | 26 ++- 2 files changed, 121 insertions(+), 113 deletions(-) diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index c658c914ee65..22f703a91e8f 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -40,19 +40,19 @@ - `1k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml`` * - 8xB200_NVL - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp8_conc32.yaml `_ @@ -70,19 +70,19 @@ - `8k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml`` * - 8xB200_NVL - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp8_conc32.yaml `_ @@ -100,19 +100,19 @@ - `1k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml`` * - 8xH200_SXM - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp8_conc32.yaml `_ @@ -130,19 +130,19 @@ - `8k1k_tp8_conc4.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp8_conc8.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp8_conc16.yaml `_ - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml`` * - 8xH200_SXM - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp8_conc32.yaml `_ @@ -187,61 +187,61 @@ - `1k1k_tp8_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp4_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp8_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp4_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp8_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp4_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp8_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 64 - `1k1k_tp4_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 64 - `1k1k_tp8_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml`` * - 4xB200_NVL - - Balanced + - Low Latency - 1024 / 1024 - 128 - `1k1k_tp4_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml`` * - 8xB200_NVL - - Balanced + - Low Latency - 1024 / 1024 - 128 - `1k1k_tp8_conc128.yaml `_ @@ -271,61 +271,61 @@ - `8k1k_tp8_conc4.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp4_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp8_conc8.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp4_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp8_conc16.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp4_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp8_conc32.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 64 - `8k1k_tp4_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 64 - `8k1k_tp8_conc64.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml`` * - 4xB200_NVL - - Balanced + - Low Latency - 8192 / 1024 - 128 - `8k1k_tp4_conc128.yaml `_ - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml`` * - 8xB200_NVL - - Balanced + - Low Latency - 8192 / 1024 - 128 - `8k1k_tp8_conc128.yaml `_ @@ -388,73 +388,73 @@ - `1k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml`` * - B200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml`` * - 2xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml`` * - B200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml`` * - 2xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml`` * - B200_NVL - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml`` * - 2xB200_NVL - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml`` * - 4xB200_NVL - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml`` * - 8xB200_NVL - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp8_conc32.yaml `_ @@ -508,73 +508,73 @@ - `1k8k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml`` * - B200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml`` * - 2xB200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml`` * - B200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml`` * - 2xB200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml`` * - B200_NVL - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml`` * - 2xB200_NVL - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml`` * - 4xB200_NVL - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml`` * - 8xB200_NVL - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp8_conc32.yaml `_ @@ -628,73 +628,73 @@ - `8k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml`` * - B200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml`` * - 2xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml`` * - B200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml`` * - 2xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml`` * - 4xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml`` * - 8xB200_NVL - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml`` * - B200_NVL - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml`` * - 2xB200_NVL - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml`` * - 4xB200_NVL - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml`` * - 8xB200_NVL - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp8_conc32.yaml `_ @@ -748,73 +748,73 @@ - `1k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml`` * - H200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml`` * - 2xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml`` * - 4xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 8 - `1k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml`` * - H200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml`` * - 2xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml`` * - 4xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 1024 / 1024 - 16 - `1k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml`` * - H200_SXM - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml`` * - 2xH200_SXM - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml`` * - 4xH200_SXM - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml`` * - 8xH200_SXM - - Balanced + - Low Latency - 1024 / 1024 - 32 - `1k1k_tp8_conc32.yaml `_ @@ -868,73 +868,73 @@ - `1k8k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml`` * - H200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml`` * - 2xH200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml`` * - 4xH200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 8 - `1k8k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml`` * - H200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml`` * - 2xH200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml`` * - 4xH200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 1024 / 8192 - 16 - `1k8k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml`` * - H200_SXM - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml`` * - 2xH200_SXM - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml`` * - 4xH200_SXM - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml`` * - 8xH200_SXM - - Balanced + - Low Latency - 1024 / 8192 - 32 - `1k8k_tp8_conc32.yaml `_ @@ -988,73 +988,73 @@ - `8k1k_tp8_conc4.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml`` * - H200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp1_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml`` * - 2xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp2_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml`` * - 4xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp4_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 8 - `8k1k_tp8_conc8.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml`` * - H200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp1_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml`` * - 2xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp2_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml`` * - 4xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp4_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml`` * - 8xH200_SXM - - Min Latency + - Low Latency - 8192 / 1024 - 16 - `8k1k_tp8_conc16.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml`` * - H200_SXM - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp1_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml`` * - 2xH200_SXM - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp2_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml`` * - 4xH200_SXM - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp4_conc32.yaml `_ - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml`` * - 8xH200_SXM - - Balanced + - Low Latency - 8192 / 1024 - 32 - `8k1k_tp8_conc32.yaml `_ diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 14ad1f5fbff9..dd9df4f5761b 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -14,6 +14,7 @@ # limitations under the License. +import importlib.util import os import sys from collections import defaultdict @@ -21,9 +22,14 @@ SCRIPT_DIR = Path(__file__).parent.resolve() REPO_ROOT = SCRIPT_DIR.parent -sys.path.insert(0, str(REPO_ROOT)) -from tensorrt_llm.configure.database import DATABASE_LIST_PATH, RecipeList # noqa: E402 +# Load database module directly to avoid tensorrt_llm.__init__ (requires torch) +_spec = importlib.util.spec_from_file_location( + "database", REPO_ROOT / "tensorrt_llm" / "configure" / "database.py" +) +_db = importlib.util.module_from_spec(_spec) +_spec.loader.exec_module(_db) +DATABASE_LIST_PATH, RecipeList = _db.DATABASE_LIST_PATH, _db.RecipeList MODEL_INFO = { "deepseek-ai/DeepSeek-R1-0528": { @@ -141,20 +147,22 @@ def generate_rst(yaml_path, output_file=None): if len(entries) == 1: if conc <= 16: - profile = "Min Latency" + profile = "Low Latency" elif conc >= 64: - profile = "Max Throughput" + profile = "High Throughput" else: profile = "Balanced" else: - relative_pos = (conc - min_conc) / conc_range if conc_range > 0 else 0.5 - - if relative_pos < 0.33: + if conc == min_conc: profile = "Min Latency" - elif relative_pos > 0.67: + elif conc == max_conc: profile = "Max Throughput" else: - profile = "Balanced" + relative_pos = (conc - min_conc) / conc_range + if relative_pos < 0.5: + profile = "Low Latency" + else: + profile = "High Throughput" full_config_path = os.path.join("tensorrt_llm/configure", config_path) command = f"trtllm-serve {model} --extra_llm_api_options ${{TRTLLM_DIR}}/{full_config_path}" From 0977e7e74af53fee82956b30b3ebf306918e05e2 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 06:56:03 -0800 Subject: [PATCH 33/39] split the note section into separate rst file Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .gitignore | 1 + .../deployment-guide/comprehensive_table.rst | 19 ++-------- ...loyment-guide-for-deepseek-r1-on-trtllm.md | 4 +++ .../deployment-guide-for-gpt-oss-on-trtllm.md | 4 +++ docs/source/deployment-guide/index.rst | 6 ++-- .../source/deployment-guide/note_sections.rst | 36 +++++++++++++++++++ scripts/generate_config_table.py | 33 +++-------------- 7 files changed, 56 insertions(+), 47 deletions(-) create mode 100644 docs/source/deployment-guide/note_sections.rst diff --git a/.gitignore b/.gitignore index bf69cca9b787..3902792b09c6 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ tensorrt_llm/scripts docs/source/**/*.rst !docs/source/examples/index.rst !docs/source/deployment-guide/comprehensive_table.rst +!docs/source/deployment-guide/note_sections.rst *.swp # Testing diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst index 22f703a91e8f..45ae52ee1a21 100644 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ b/docs/source/deployment-guide/comprehensive_table.rst @@ -1,19 +1,6 @@ -.. note:: - - **Traffic Patterns**: The ISL (Input Sequence Length) and OSL (Output Sequence Length) - values in each configuration represent the **maximum supported values** for that config. - Requests exceeding these limits may result in errors. - - To handle requests with input sequences **longer than the configured ISL**, add the following - to your config file: - - .. code-block:: yaml - - enable_chunked_prefill: true - - This enables chunked prefill, which processes long input sequences in chunks rather than - requiring them to fit within a single prefill operation. Note that enabling chunked prefill - does **not** guarantee optimal performance—these configs are tuned for the specified ISL/OSL. +.. include:: note_sections.rst + :start-after: .. start-note-traffic-patterns + :end-before: .. end-note-traffic-patterns .. start-deepseek-ai/DeepSeek-R1-0528 diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index 8386a7c98d1e..c40aad0f9948 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -435,6 +435,10 @@ $$ The following tables list recommended configurations from the comprehensive database for different performance profiles. ```{eval-rst} +.. include:: note_sections.rst + :start-after: .. start-note-traffic-patterns + :end-before: .. end-note-traffic-patterns + .. include:: comprehensive_table.rst :start-after: .. start-deepseek-ai/DeepSeek-R1-0528 :end-before: .. end-deepseek-ai/DeepSeek-R1-0528 diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index 34e097f350cf..5fd67b07b7cb 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -383,6 +383,10 @@ $$ The following table lists recommended configurations from the comprehensive database for different performance profiles. ```{eval-rst} +.. include:: note_sections.rst + :start-after: .. start-note-traffic-patterns + :end-before: .. end-note-traffic-patterns + .. include:: comprehensive_table.rst :start-after: .. start-openai/gpt-oss-120b :end-before: .. end-openai/gpt-oss-120b diff --git a/docs/source/deployment-guide/index.rst b/docs/source/deployment-guide/index.rst index f4f090aeac66..e396637ac172 100644 --- a/docs/source/deployment-guide/index.rst +++ b/docs/source/deployment-guide/index.rst @@ -17,9 +17,9 @@ The TensorRT LLM Docker container makes these config files available at ``/app/t export TRTLLM_DIR="/app/tensorrt_llm" # path to the TensorRT LLM repo in your local environment -.. note:: - - The configs here are specifically optimized for a target ISL/OSL (Input/Output Sequence Length) of 1024/1024. If your traffic pattern is different, refer to the :ref:`Comprehensive Configuration Database` section below which covers a larger set of traffic patterns and performance profiles. +.. include:: note_sections.rst + :start-after: .. start-note-quick-start-isl-osl + :end-before: .. end-note-quick-start-isl-osl This table is designed to provide a straightforward starting point; for detailed model-specific deployment guides, check out the guides below. diff --git a/docs/source/deployment-guide/note_sections.rst b/docs/source/deployment-guide/note_sections.rst new file mode 100644 index 000000000000..4cd0d1c41dda --- /dev/null +++ b/docs/source/deployment-guide/note_sections.rst @@ -0,0 +1,36 @@ +.. + Reusable note sections for deployment guides. + Include specific notes using: + + .. include:: note_sections.rst + :start-after: .. start-note- + :end-before: .. end-note- + +.. start-note-traffic-patterns + +.. note:: + + **Traffic Patterns**: The ISL (Input Sequence Length) and OSL (Output Sequence Length) + values in each configuration represent the **maximum supported values** for that config. + Requests exceeding these limits may result in errors. + + To handle requests with input sequences **longer than the configured ISL**, add the following + to your config file: + + .. code-block:: yaml + + enable_chunked_prefill: true + + This enables chunked prefill, which processes long input sequences in chunks rather than + requiring them to fit within a single prefill operation. Note that enabling chunked prefill + does **not** guarantee optimal performance—these configs are tuned for the specified ISL/OSL. + +.. end-note-traffic-patterns + +.. start-note-quick-start-isl-osl + +.. note:: + + The configs here are specifically optimized for a target ISL/OSL (Input/Output Sequence Length) of 1024/1024. If your traffic pattern is different, refer to the :ref:`Comprehensive Configuration Database` section below which covers a larger set of traffic patterns and performance profiles. + +.. end-note-quick-start-isl-osl diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index dd9df4f5761b..7441f7ab4a18 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -63,34 +63,11 @@ def generate_rst(yaml_path, output_file=None): model_groups[recipe.model][key].append(recipe) lines = [] - lines.append(".. note::") - lines.append("") - lines.append( - " **Traffic Patterns**: The ISL (Input Sequence Length) and OSL (Output Sequence Length)" - ) - lines.append( - " values in each configuration represent the **maximum supported values** for that config." - ) - lines.append(" Requests exceeding these limits may result in errors.") - lines.append("") - lines.append( - " To handle requests with input sequences **longer than the configured ISL**, add the following" - ) - lines.append(" to your config file:") - lines.append("") - lines.append(" .. code-block:: yaml") - lines.append("") - lines.append(" enable_chunked_prefill: true") - lines.append("") - lines.append( - " This enables chunked prefill, which processes long input sequences in chunks rather than" - ) - lines.append( - " requiring them to fit within a single prefill operation. Note that enabling chunked prefill" - ) - lines.append( - " does **not** guarantee optimal performance—these configs are tuned for the specified ISL/OSL." - ) + + # Include note_sections.rst at the top (relative include for Sphinx) + lines.append(".. include:: note_sections.rst") + lines.append(" :start-after: .. start-note-traffic-patterns") + lines.append(" :end-before: .. end-note-traffic-patterns") lines.append("") sorted_models = sorted(model_groups.keys()) From ee18a55ca36655ad5d0b47b787b19f6b57321325 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 08:25:00 -0800 Subject: [PATCH 34/39] move config db to examples Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .gitignore | 2 +- .../deployment-guide/comprehensive_table.rst | 1074 ----------------- docs/source/deployment-guide/config_table.rst | 1074 +++++++++++++++++ ...loyment-guide-for-deepseek-r1-on-trtllm.md | 4 +- .../deployment-guide-for-gpt-oss-on-trtllm.md | 2 +- docs/source/deployment-guide/index.rst | 48 +- .../{ => curated}/deepseek-r1-deepgemm.yaml | 0 .../{ => curated}/deepseek-r1-latency.yaml | 0 .../{ => curated}/deepseek-r1-throughput.yaml | 0 .../{ => curated}/gpt-oss-120b-latency.yaml | 0 .../gpt-oss-120b-throughput.yaml | 0 .../configs/{ => curated}/llama-3.3-70b.yaml | 0 .../configs/{ => curated}/llama-4-scout.yaml | 0 .../{ => curated}/qwen3-disagg-prefill.yaml | 0 .../configs/{ => curated}/qwen3-next.yaml | 0 examples/configs/{ => curated}/qwen3.yaml | 0 .../configs/database}/database.py | 2 +- .../B200/1k1k_tp8_conc16.yaml | 0 .../B200/1k1k_tp8_conc32.yaml | 0 .../DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml | 0 .../B200/1k1k_tp8_conc64.yaml | 0 .../DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml | 0 .../B200/8k1k_tp8_conc16.yaml | 0 .../B200/8k1k_tp8_conc32.yaml | 0 .../DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml | 0 .../B200/8k1k_tp8_conc64.yaml | 0 .../DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml | 0 .../H200/1k1k_tp8_conc16.yaml | 0 .../H200/1k1k_tp8_conc32.yaml | 0 .../DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml | 0 .../H200/1k1k_tp8_conc64.yaml | 0 .../DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml | 0 .../H200/8k1k_tp8_conc16.yaml | 0 .../H200/8k1k_tp8_conc32.yaml | 0 .../DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml | 0 .../H200/8k1k_tp8_conc64.yaml | 0 .../DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml | 0 .../configs/database/lookup.yaml | 336 +++--- .../B200/1k1k_tp4_conc128.yaml | 0 .../B200/1k1k_tp4_conc16.yaml | 0 .../B200/1k1k_tp4_conc256.yaml | 0 .../B200/1k1k_tp4_conc32.yaml | 0 .../B200/1k1k_tp4_conc4.yaml | 0 .../B200/1k1k_tp4_conc64.yaml | 0 .../B200/1k1k_tp4_conc8.yaml | 0 .../B200/1k1k_tp8_conc128.yaml | 0 .../B200/1k1k_tp8_conc16.yaml | 0 .../B200/1k1k_tp8_conc256.yaml | 0 .../B200/1k1k_tp8_conc32.yaml | 0 .../B200/1k1k_tp8_conc4.yaml | 0 .../B200/1k1k_tp8_conc64.yaml | 0 .../B200/1k1k_tp8_conc8.yaml | 0 .../B200/8k1k_tp4_conc128.yaml | 0 .../B200/8k1k_tp4_conc16.yaml | 0 .../B200/8k1k_tp4_conc256.yaml | 0 .../B200/8k1k_tp4_conc32.yaml | 0 .../B200/8k1k_tp4_conc4.yaml | 0 .../B200/8k1k_tp4_conc64.yaml | 0 .../B200/8k1k_tp4_conc8.yaml | 0 .../B200/8k1k_tp8_conc128.yaml | 0 .../B200/8k1k_tp8_conc16.yaml | 0 .../B200/8k1k_tp8_conc256.yaml | 0 .../B200/8k1k_tp8_conc32.yaml | 0 .../B200/8k1k_tp8_conc4.yaml | 0 .../B200/8k1k_tp8_conc64.yaml | 0 .../B200/8k1k_tp8_conc8.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp1_conc16.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp1_conc32.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp1_conc4.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp1_conc64.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp1_conc8.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp2_conc16.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp2_conc32.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp2_conc4.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp2_conc64.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp2_conc8.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp4_conc16.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp4_conc32.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp4_conc4.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp4_conc64.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp4_conc8.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp8_conc16.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp8_conc32.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp8_conc4.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp8_conc64.yaml | 0 .../gpt-oss-120b/B200/1k1k_tp8_conc8.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp1_conc16.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp1_conc32.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp1_conc4.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp1_conc64.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp1_conc8.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp2_conc16.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp2_conc32.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp2_conc4.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp2_conc64.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp2_conc8.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp4_conc16.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp4_conc32.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp4_conc4.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp4_conc64.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp4_conc8.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp8_conc16.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp8_conc32.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp8_conc4.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp8_conc64.yaml | 0 .../gpt-oss-120b/B200/1k8k_tp8_conc8.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp1_conc16.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp1_conc32.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp1_conc4.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp1_conc64.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp1_conc8.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp2_conc16.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp2_conc32.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp2_conc4.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp2_conc64.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp2_conc8.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp4_conc16.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp4_conc32.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp4_conc4.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp4_conc64.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp4_conc8.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp8_conc16.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp8_conc32.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp8_conc4.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp8_conc64.yaml | 0 .../gpt-oss-120b/B200/8k1k_tp8_conc8.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp1_conc16.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp1_conc32.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp1_conc4.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp1_conc64.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp1_conc8.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp2_conc16.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp2_conc32.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp2_conc4.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp2_conc64.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp2_conc8.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp4_conc16.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp4_conc32.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp4_conc4.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp4_conc64.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp4_conc8.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp8_conc16.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp8_conc32.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp8_conc4.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp8_conc64.yaml | 0 .../gpt-oss-120b/H200/1k1k_tp8_conc8.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp1_conc16.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp1_conc32.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp1_conc4.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp1_conc64.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp1_conc8.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp2_conc16.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp2_conc32.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp2_conc4.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp2_conc64.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp2_conc8.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp4_conc16.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp4_conc32.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp4_conc4.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp4_conc64.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp4_conc8.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp8_conc16.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp8_conc32.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp8_conc4.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp8_conc64.yaml | 0 .../gpt-oss-120b/H200/1k8k_tp8_conc8.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp1_conc16.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp1_conc32.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp1_conc4.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp1_conc64.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp1_conc8.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp2_conc16.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp2_conc32.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp2_conc4.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp2_conc64.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp2_conc8.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp4_conc16.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp4_conc32.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp4_conc4.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp4_conc64.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp4_conc8.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp8_conc16.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp8_conc32.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp8_conc4.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp8_conc64.yaml | 0 .../gpt-oss-120b/H200/8k1k_tp8_conc8.yaml | 0 scripts/generate_config_table.py | 37 +- setup.py | 1 - tensorrt_llm/configure/__init__.py | 0 .../tools/test_generate_config_table.py | 8 +- 190 files changed, 1292 insertions(+), 1296 deletions(-) delete mode 100644 docs/source/deployment-guide/comprehensive_table.rst create mode 100644 docs/source/deployment-guide/config_table.rst rename examples/configs/{ => curated}/deepseek-r1-deepgemm.yaml (100%) rename examples/configs/{ => curated}/deepseek-r1-latency.yaml (100%) rename examples/configs/{ => curated}/deepseek-r1-throughput.yaml (100%) rename examples/configs/{ => curated}/gpt-oss-120b-latency.yaml (100%) rename examples/configs/{ => curated}/gpt-oss-120b-throughput.yaml (100%) rename examples/configs/{ => curated}/llama-3.3-70b.yaml (100%) rename examples/configs/{ => curated}/llama-4-scout.yaml (100%) rename examples/configs/{ => curated}/qwen3-disagg-prefill.yaml (100%) rename examples/configs/{ => curated}/qwen3-next.yaml (100%) rename examples/configs/{ => curated}/qwen3.yaml (100%) rename {tensorrt_llm/configure => examples/configs/database}/database.py (96%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml (100%) rename tensorrt_llm/configure/database/scenario_list.yaml => examples/configs/database/lookup.yaml (53%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml (100%) rename {tensorrt_llm/configure => examples/configs}/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml (100%) delete mode 100644 tensorrt_llm/configure/__init__.py diff --git a/.gitignore b/.gitignore index 3902792b09c6..85ca98724dc2 100644 --- a/.gitignore +++ b/.gitignore @@ -55,7 +55,7 @@ tensorrt_llm/scripts *docs/source/_cpp_gen* docs/source/**/*.rst !docs/source/examples/index.rst -!docs/source/deployment-guide/comprehensive_table.rst +!docs/source/deployment-guide/config_table.rst !docs/source/deployment-guide/note_sections.rst *.swp diff --git a/docs/source/deployment-guide/comprehensive_table.rst b/docs/source/deployment-guide/comprehensive_table.rst deleted file mode 100644 index 45ae52ee1a21..000000000000 --- a/docs/source/deployment-guide/comprehensive_table.rst +++ /dev/null @@ -1,1074 +0,0 @@ -.. include:: note_sections.rst - :start-after: .. start-note-traffic-patterns - :end-before: .. end-note-traffic-patterns - -.. start-deepseek-ai/DeepSeek-R1-0528 - -.. _deepseek-ai/DeepSeek-R1-0528: - -`DeepSeek-R1 `_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :width: 100% - :header-rows: 1 - :widths: 12 15 15 13 20 25 - - * - GPU - - Performance Profile - - ISL / OSL - - Concurrency - - Config - - Command - * - 8xB200_NVL - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml`` - * - 8xB200_NVL - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml`` - * - 8xB200_NVL - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml`` - * - 8xB200_NVL - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml`` - * - 8xH200_SXM - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml`` - * - 8xH200_SXM - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml`` - * - 8xH200_SXM - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml`` - * - 8xH200_SXM - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml`` - * - 8xH200_SXM - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml`` - * - 8xH200_SXM - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml`` - * - 8xH200_SXM - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml`` - -.. end-deepseek-ai/DeepSeek-R1-0528 - -.. start-nvidia/DeepSeek-R1-0528-FP4-v2 - -.. _nvidia/DeepSeek-R1-0528-FP4-v2: - -`DeepSeek-R1 (NVFP4) `_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :width: 100% - :header-rows: 1 - :widths: 12 15 15 13 20 25 - - * - GPU - - Performance Profile - - ISL / OSL - - Concurrency - - Config - - Command - * - 4xB200_NVL - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml`` - * - 8xB200_NVL - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 64 - - `1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 64 - - `1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 128 - - `1k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 128 - - `1k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml`` - * - 4xB200_NVL - - Max Throughput - - 1024 / 1024 - - 256 - - `1k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml`` - * - 8xB200_NVL - - Max Throughput - - 1024 / 1024 - - 256 - - `1k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml`` - * - 4xB200_NVL - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml`` - * - 8xB200_NVL - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 64 - - `8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 64 - - `8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 128 - - `8k1k_tp4_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 128 - - `8k1k_tp8_conc128.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml`` - * - 4xB200_NVL - - Max Throughput - - 8192 / 1024 - - 256 - - `8k1k_tp4_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml`` - * - 8xB200_NVL - - Max Throughput - - 8192 / 1024 - - 256 - - `8k1k_tp8_conc256.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml`` - -.. end-nvidia/DeepSeek-R1-0528-FP4-v2 - -.. start-openai/gpt-oss-120b - -.. _openai/gpt-oss-120b: - -`gpt-oss-120b `_ -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ - -.. list-table:: - :width: 100% - :header-rows: 1 - :widths: 12 15 15 13 20 25 - - * - GPU - - Performance Profile - - ISL / OSL - - Concurrency - - Config - - Command - * - B200_NVL - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml`` - * - 2xB200_NVL - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml`` - * - 4xB200_NVL - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml`` - * - 8xB200_NVL - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml`` - * - B200_NVL - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml`` - * - 2xB200_NVL - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml`` - * - B200_NVL - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml`` - * - 2xB200_NVL - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml`` - * - B200_NVL - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml`` - * - 2xB200_NVL - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml`` - * - B200_NVL - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml`` - * - 2xB200_NVL - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml`` - * - 4xB200_NVL - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml`` - * - 8xB200_NVL - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml`` - * - B200_NVL - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml`` - * - 2xB200_NVL - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml`` - * - 4xB200_NVL - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml`` - * - 8xB200_NVL - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml`` - * - B200_NVL - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml`` - * - 2xB200_NVL - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml`` - * - B200_NVL - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml`` - * - 2xB200_NVL - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml`` - * - B200_NVL - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml`` - * - 2xB200_NVL - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml`` - * - 4xB200_NVL - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml`` - * - 8xB200_NVL - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml`` - * - B200_NVL - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml`` - * - 2xB200_NVL - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml`` - * - 4xB200_NVL - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml`` - * - 8xB200_NVL - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml`` - * - B200_NVL - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml`` - * - 2xB200_NVL - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml`` - * - 4xB200_NVL - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml`` - * - 8xB200_NVL - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml`` - * - B200_NVL - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml`` - * - 2xB200_NVL - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml`` - * - B200_NVL - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml`` - * - 2xB200_NVL - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml`` - * - B200_NVL - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml`` - * - 2xB200_NVL - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml`` - * - 4xB200_NVL - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml`` - * - 8xB200_NVL - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml`` - * - B200_NVL - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml`` - * - 2xB200_NVL - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml`` - * - 4xB200_NVL - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml`` - * - 8xB200_NVL - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml`` - * - H200_SXM - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml`` - * - 2xH200_SXM - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml`` - * - 4xH200_SXM - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml`` - * - 8xH200_SXM - - Min Latency - - 1024 / 1024 - - 4 - - `1k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml`` - * - H200_SXM - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml`` - * - 2xH200_SXM - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml`` - * - 4xH200_SXM - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 1024 - - 8 - - `1k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml`` - * - H200_SXM - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml`` - * - 2xH200_SXM - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml`` - * - 4xH200_SXM - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 1024 - - 16 - - `1k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml`` - * - H200_SXM - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml`` - * - 2xH200_SXM - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml`` - * - 4xH200_SXM - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 1024 - - 32 - - `1k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml`` - * - H200_SXM - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml`` - * - 2xH200_SXM - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml`` - * - 4xH200_SXM - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml`` - * - 8xH200_SXM - - Max Throughput - - 1024 / 1024 - - 64 - - `1k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml`` - * - H200_SXM - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml`` - * - 2xH200_SXM - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml`` - * - 4xH200_SXM - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml`` - * - 8xH200_SXM - - Min Latency - - 1024 / 8192 - - 4 - - `1k8k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml`` - * - H200_SXM - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml`` - * - 2xH200_SXM - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml`` - * - 4xH200_SXM - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 8192 - - 8 - - `1k8k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml`` - * - H200_SXM - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml`` - * - 2xH200_SXM - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml`` - * - 4xH200_SXM - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 8192 - - 16 - - `1k8k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml`` - * - H200_SXM - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml`` - * - 2xH200_SXM - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml`` - * - 4xH200_SXM - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml`` - * - 8xH200_SXM - - Low Latency - - 1024 / 8192 - - 32 - - `1k8k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml`` - * - H200_SXM - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml`` - * - 2xH200_SXM - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml`` - * - 4xH200_SXM - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml`` - * - 8xH200_SXM - - Max Throughput - - 1024 / 8192 - - 64 - - `1k8k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml`` - * - H200_SXM - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp1_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml`` - * - 2xH200_SXM - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp2_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml`` - * - 4xH200_SXM - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp4_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml`` - * - 8xH200_SXM - - Min Latency - - 8192 / 1024 - - 4 - - `8k1k_tp8_conc4.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml`` - * - H200_SXM - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp1_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml`` - * - 2xH200_SXM - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp2_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml`` - * - 4xH200_SXM - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp4_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml`` - * - 8xH200_SXM - - Low Latency - - 8192 / 1024 - - 8 - - `8k1k_tp8_conc8.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml`` - * - H200_SXM - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp1_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml`` - * - 2xH200_SXM - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp2_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml`` - * - 4xH200_SXM - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp4_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml`` - * - 8xH200_SXM - - Low Latency - - 8192 / 1024 - - 16 - - `8k1k_tp8_conc16.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml`` - * - H200_SXM - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp1_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml`` - * - 2xH200_SXM - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp2_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml`` - * - 4xH200_SXM - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp4_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml`` - * - 8xH200_SXM - - Low Latency - - 8192 / 1024 - - 32 - - `8k1k_tp8_conc32.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml`` - * - H200_SXM - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp1_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml`` - * - 2xH200_SXM - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp2_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml`` - * - 4xH200_SXM - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp4_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml`` - * - 8xH200_SXM - - Max Throughput - - 8192 / 1024 - - 64 - - `8k1k_tp8_conc64.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml`` - -.. end-openai/gpt-oss-120b diff --git a/docs/source/deployment-guide/config_table.rst b/docs/source/deployment-guide/config_table.rst new file mode 100644 index 000000000000..d28fed25a8ea --- /dev/null +++ b/docs/source/deployment-guide/config_table.rst @@ -0,0 +1,1074 @@ +.. include:: note_sections.rst + :start-after: .. start-note-traffic-patterns + :end-before: .. end-note-traffic-patterns + +.. start-deepseek-ai/DeepSeek-R1-0528 + +.. _deepseek-ai/DeepSeek-R1-0528: + +`DeepSeek-R1 `_ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :width: 100% + :header-rows: 1 + :widths: 12 15 15 13 20 25 + + * - GPU + - Performance Profile + - ISL / OSL + - Concurrency + - Config + - Command + * - 8xB200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml`` + * - 8xB200_NVL + - Balanced + - 1024 / 1024 + - 16 + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml`` + * - 8xB200_NVL + - Max Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml`` + * - 8xB200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml`` + * - 8xB200_NVL + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml`` + * - 8xB200_NVL + - Balanced + - 8192 / 1024 + - 16 + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml`` + * - 8xB200_NVL + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml`` + * - 8xB200_NVL + - Max Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml`` + * - 8xH200_SXM + - Min Latency + - 1024 / 1024 + - 4 + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml`` + * - 8xH200_SXM + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml`` + * - 8xH200_SXM + - Balanced + - 1024 / 1024 + - 16 + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml`` + * - 8xH200_SXM + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml`` + * - 8xH200_SXM + - Max Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml`` + * - 8xH200_SXM + - Min Latency + - 8192 / 1024 + - 4 + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml`` + * - 8xH200_SXM + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml`` + * - 8xH200_SXM + - Balanced + - 8192 / 1024 + - 16 + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml`` + * - 8xH200_SXM + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml`` + * - 8xH200_SXM + - Max Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml`` + +.. end-deepseek-ai/DeepSeek-R1-0528 + +.. start-nvidia/DeepSeek-R1-0528-FP4-v2 + +.. _nvidia/DeepSeek-R1-0528-FP4-v2: + +`DeepSeek-R1 (NVFP4) `_ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :width: 100% + :header-rows: 1 + :widths: 12 15 15 13 20 25 + + * - GPU + - Performance Profile + - ISL / OSL + - Concurrency + - Config + - Command + * - 4xB200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - `1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 1024 + - 4 + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml`` + * - 4xB200_NVL + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml`` + * - 4xB200_NVL + - Low Latency + - 1024 / 1024 + - 16 + - `1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 1024 + - 16 + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml`` + * - 4xB200_NVL + - Low Latency + - 1024 / 1024 + - 32 + - `1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 1024 + - 128 + - `1k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 1024 + - 128 + - `1k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 1024 + - 256 + - `1k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml`` + * - 8xB200_NVL + - Max Throughput + - 1024 / 1024 + - 256 + - `1k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml`` + * - 4xB200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - `8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml`` + * - 8xB200_NVL + - Low Latency + - 8192 / 1024 + - 4 + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml`` + * - 4xB200_NVL + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml`` + * - 8xB200_NVL + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml`` + * - 4xB200_NVL + - Low Latency + - 8192 / 1024 + - 16 + - `8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml`` + * - 8xB200_NVL + - Low Latency + - 8192 / 1024 + - 16 + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml`` + * - 4xB200_NVL + - Low Latency + - 8192 / 1024 + - 32 + - `8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml`` + * - 8xB200_NVL + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml`` + * - 4xB200_NVL + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml`` + * - 8xB200_NVL + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml`` + * - 4xB200_NVL + - High Throughput + - 8192 / 1024 + - 128 + - `8k1k_tp4_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml`` + * - 8xB200_NVL + - High Throughput + - 8192 / 1024 + - 128 + - `8k1k_tp8_conc128.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml`` + * - 4xB200_NVL + - High Throughput + - 8192 / 1024 + - 256 + - `8k1k_tp4_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml`` + * - 8xB200_NVL + - Max Throughput + - 8192 / 1024 + - 256 + - `8k1k_tp8_conc256.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-0528-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml`` + +.. end-nvidia/DeepSeek-R1-0528-FP4-v2 + +.. start-openai/gpt-oss-120b + +.. _openai/gpt-oss-120b: + +`gpt-oss-120b `_ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :width: 100% + :header-rows: 1 + :widths: 12 15 15 13 20 25 + + * - GPU + - Performance Profile + - ISL / OSL + - Concurrency + - Config + - Command + * - B200_NVL + - Min Latency + - 1024 / 1024 + - 4 + - `1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml`` + * - 2xB200_NVL + - Low Latency + - 1024 / 1024 + - 4 + - `1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml`` + * - 4xB200_NVL + - Low Latency + - 1024 / 1024 + - 4 + - `1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 1024 + - 4 + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml`` + * - B200_NVL + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml`` + * - 2xB200_NVL + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml`` + * - 4xB200_NVL + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml`` + * - B200_NVL + - Low Latency + - 1024 / 1024 + - 16 + - `1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml`` + * - 2xB200_NVL + - Low Latency + - 1024 / 1024 + - 16 + - `1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 1024 + - 16 + - `1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 1024 + - 16 + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml`` + * - B200_NVL + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml`` + * - 2xB200_NVL + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml`` + * - B200_NVL + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml`` + * - 2xB200_NVL + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml`` + * - 8xB200_NVL + - Max Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml`` + * - B200_NVL + - Min Latency + - 1024 / 8192 + - 4 + - `1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml`` + * - 2xB200_NVL + - Low Latency + - 1024 / 8192 + - 4 + - `1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml`` + * - 4xB200_NVL + - Low Latency + - 1024 / 8192 + - 4 + - `1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 8192 + - 4 + - `1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml`` + * - B200_NVL + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml`` + * - 2xB200_NVL + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml`` + * - 4xB200_NVL + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml`` + * - 8xB200_NVL + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml`` + * - B200_NVL + - Low Latency + - 1024 / 8192 + - 16 + - `1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml`` + * - 2xB200_NVL + - Low Latency + - 1024 / 8192 + - 16 + - `1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 8192 + - 16 + - `1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 8192 + - 16 + - `1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml`` + * - B200_NVL + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml`` + * - 2xB200_NVL + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml`` + * - 8xB200_NVL + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml`` + * - B200_NVL + - High Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml`` + * - 2xB200_NVL + - High Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml`` + * - 4xB200_NVL + - High Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml`` + * - 8xB200_NVL + - Max Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml`` + * - B200_NVL + - Min Latency + - 8192 / 1024 + - 4 + - `8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml`` + * - 2xB200_NVL + - Low Latency + - 8192 / 1024 + - 4 + - `8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml`` + * - 4xB200_NVL + - Low Latency + - 8192 / 1024 + - 4 + - `8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml`` + * - 8xB200_NVL + - Low Latency + - 8192 / 1024 + - 4 + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml`` + * - B200_NVL + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml`` + * - 2xB200_NVL + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml`` + * - 4xB200_NVL + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml`` + * - 8xB200_NVL + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml`` + * - B200_NVL + - Low Latency + - 8192 / 1024 + - 16 + - `8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml`` + * - 2xB200_NVL + - Low Latency + - 8192 / 1024 + - 16 + - `8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml`` + * - 4xB200_NVL + - High Throughput + - 8192 / 1024 + - 16 + - `8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml`` + * - 8xB200_NVL + - High Throughput + - 8192 / 1024 + - 16 + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml`` + * - B200_NVL + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml`` + * - 2xB200_NVL + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml`` + * - 4xB200_NVL + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml`` + * - 8xB200_NVL + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml`` + * - B200_NVL + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml`` + * - 2xB200_NVL + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml`` + * - 4xB200_NVL + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml`` + * - 8xB200_NVL + - Max Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml`` + * - H200_SXM + - Min Latency + - 1024 / 1024 + - 4 + - `1k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml`` + * - 2xH200_SXM + - Low Latency + - 1024 / 1024 + - 4 + - `1k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml`` + * - 4xH200_SXM + - Low Latency + - 1024 / 1024 + - 4 + - `1k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml`` + * - 8xH200_SXM + - Low Latency + - 1024 / 1024 + - 4 + - `1k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml`` + * - H200_SXM + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml`` + * - 2xH200_SXM + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml`` + * - 4xH200_SXM + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml`` + * - 8xH200_SXM + - Low Latency + - 1024 / 1024 + - 8 + - `1k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml`` + * - H200_SXM + - Low Latency + - 1024 / 1024 + - 16 + - `1k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml`` + * - 2xH200_SXM + - Low Latency + - 1024 / 1024 + - 16 + - `1k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml`` + * - 4xH200_SXM + - High Throughput + - 1024 / 1024 + - 16 + - `1k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml`` + * - 8xH200_SXM + - High Throughput + - 1024 / 1024 + - 16 + - `1k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml`` + * - H200_SXM + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml`` + * - 2xH200_SXM + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml`` + * - 4xH200_SXM + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml`` + * - 8xH200_SXM + - High Throughput + - 1024 / 1024 + - 32 + - `1k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml`` + * - H200_SXM + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml`` + * - 2xH200_SXM + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml`` + * - 4xH200_SXM + - High Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml`` + * - 8xH200_SXM + - Max Throughput + - 1024 / 1024 + - 64 + - `1k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml`` + * - H200_SXM + - Min Latency + - 1024 / 8192 + - 4 + - `1k8k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml`` + * - 2xH200_SXM + - Low Latency + - 1024 / 8192 + - 4 + - `1k8k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml`` + * - 4xH200_SXM + - Low Latency + - 1024 / 8192 + - 4 + - `1k8k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml`` + * - 8xH200_SXM + - Low Latency + - 1024 / 8192 + - 4 + - `1k8k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml`` + * - H200_SXM + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml`` + * - 2xH200_SXM + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml`` + * - 4xH200_SXM + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml`` + * - 8xH200_SXM + - Low Latency + - 1024 / 8192 + - 8 + - `1k8k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml`` + * - H200_SXM + - Low Latency + - 1024 / 8192 + - 16 + - `1k8k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml`` + * - 2xH200_SXM + - Low Latency + - 1024 / 8192 + - 16 + - `1k8k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml`` + * - 4xH200_SXM + - High Throughput + - 1024 / 8192 + - 16 + - `1k8k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml`` + * - 8xH200_SXM + - High Throughput + - 1024 / 8192 + - 16 + - `1k8k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml`` + * - H200_SXM + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml`` + * - 2xH200_SXM + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml`` + * - 4xH200_SXM + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml`` + * - 8xH200_SXM + - High Throughput + - 1024 / 8192 + - 32 + - `1k8k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml`` + * - H200_SXM + - High Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml`` + * - 2xH200_SXM + - High Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml`` + * - 4xH200_SXM + - High Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml`` + * - 8xH200_SXM + - Max Throughput + - 1024 / 8192 + - 64 + - `1k8k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml`` + * - H200_SXM + - Min Latency + - 8192 / 1024 + - 4 + - `8k1k_tp1_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml`` + * - 2xH200_SXM + - Low Latency + - 8192 / 1024 + - 4 + - `8k1k_tp2_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml`` + * - 4xH200_SXM + - Low Latency + - 8192 / 1024 + - 4 + - `8k1k_tp4_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml`` + * - 8xH200_SXM + - Low Latency + - 8192 / 1024 + - 4 + - `8k1k_tp8_conc4.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml`` + * - H200_SXM + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp1_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml`` + * - 2xH200_SXM + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp2_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml`` + * - 4xH200_SXM + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp4_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml`` + * - 8xH200_SXM + - Low Latency + - 8192 / 1024 + - 8 + - `8k1k_tp8_conc8.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml`` + * - H200_SXM + - Low Latency + - 8192 / 1024 + - 16 + - `8k1k_tp1_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml`` + * - 2xH200_SXM + - Low Latency + - 8192 / 1024 + - 16 + - `8k1k_tp2_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml`` + * - 4xH200_SXM + - High Throughput + - 8192 / 1024 + - 16 + - `8k1k_tp4_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml`` + * - 8xH200_SXM + - High Throughput + - 8192 / 1024 + - 16 + - `8k1k_tp8_conc16.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml`` + * - H200_SXM + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp1_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml`` + * - 2xH200_SXM + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp2_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml`` + * - 4xH200_SXM + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp4_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml`` + * - 8xH200_SXM + - High Throughput + - 8192 / 1024 + - 32 + - `8k1k_tp8_conc32.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml`` + * - H200_SXM + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp1_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml`` + * - 2xH200_SXM + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp2_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml`` + * - 4xH200_SXM + - High Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp4_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml`` + * - 8xH200_SXM + - Max Throughput + - 8192 / 1024 + - 64 + - `8k1k_tp8_conc64.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml`` + +.. end-openai/gpt-oss-120b diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index c40aad0f9948..07b86fd0ca84 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -439,13 +439,13 @@ The following tables list recommended configurations from the comprehensive data :start-after: .. start-note-traffic-patterns :end-before: .. end-note-traffic-patterns -.. include:: comprehensive_table.rst +.. include:: config_table.rst :start-after: .. start-deepseek-ai/DeepSeek-R1-0528 :end-before: .. end-deepseek-ai/DeepSeek-R1-0528 ``` ```{eval-rst} -.. include:: comprehensive_table.rst +.. include:: config_table.rst :start-after: .. start-nvidia/DeepSeek-R1-0528-FP4-v2 :end-before: .. end-nvidia/DeepSeek-R1-0528-FP4-v2 ``` diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index 5fd67b07b7cb..1e181c7ad08a 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -387,7 +387,7 @@ The following table lists recommended configurations from the comprehensive data :start-after: .. start-note-traffic-patterns :end-before: .. end-note-traffic-patterns -.. include:: comprehensive_table.rst +.. include:: config_table.rst :start-after: .. start-openai/gpt-oss-120b :end-before: .. end-openai/gpt-oss-120b ``` diff --git a/docs/source/deployment-guide/index.rst b/docs/source/deployment-guide/index.rst index e396637ac172..644a9d9ae95e 100644 --- a/docs/source/deployment-guide/index.rst +++ b/docs/source/deployment-guide/index.rst @@ -8,10 +8,10 @@ The table below contains ``trtllm-serve`` commands that can be used to easily de We maintain LLM API configuration files for these models containing recommended performance settings in two locations: -* **Curated Examples**: `examples/configs `_ - Hand-picked configurations for common scenarios. -* **Comprehensive Database**: `tensorrt_llm/configure/database `_ - A more comprehensive set of known-good configurations for various GPUs and inference scenarios. +* **Curated Examples**: `examples/configs/curated `_ - Hand-picked configurations for common scenarios. +* **Comprehensive Database**: `examples/configs/database `_ - A more comprehensive set of known-good configurations for various GPUs and traffic patterns. -The TensorRT LLM Docker container makes these config files available at ``/app/tensorrt_llm/examples/configs`` and ``/app/tensorrt_llm/tensorrt_llm/configure/database`` respectively. You can reference them as needed: +The TensorRT LLM Docker container makes these config files available at ``/app/tensorrt_llm/examples/configs/curated`` and ``/app/tensorrt_llm/examples/configs/database`` respectively. You can reference them as needed: .. code-block:: bash @@ -35,53 +35,53 @@ This table is designed to provide a straightforward starting point; for detailed * - `DeepSeek-R1 `_ - H100, H200 - Max Throughput - - `deepseek-r1-throughput.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-throughput.yaml`` + - `deepseek-r1-throughput.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-throughput.yaml`` * - `DeepSeek-R1 `_ - B200, GB200 - Max Throughput - - `deepseek-r1-deepgemm.yaml `_ - - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-deepgemm.yaml`` + - `deepseek-r1-deepgemm.yaml `_ + - ``trtllm-serve deepseek-ai/DeepSeek-R1-0528 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-deepgemm.yaml`` * - `DeepSeek-R1 (NVFP4) `_ - B200, GB200 - Max Throughput - - `deepseek-r1-throughput.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-FP4 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-throughput.yaml`` + - `deepseek-r1-throughput.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-FP4 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-throughput.yaml`` * - `DeepSeek-R1 (NVFP4) `_ - B200, GB200 - Min Latency - - `deepseek-r1-latency.yaml `_ - - ``trtllm-serve nvidia/DeepSeek-R1-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/deepseek-r1-latency.yaml`` + - `deepseek-r1-latency.yaml `_ + - ``trtllm-serve nvidia/DeepSeek-R1-FP4-v2 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-latency.yaml`` * - `gpt-oss-120b `_ - Any - Max Throughput - - `gpt-oss-120b-throughput.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/gpt-oss-120b-throughput.yaml`` + - `gpt-oss-120b-throughput.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-throughput.yaml`` * - `gpt-oss-120b `_ - Any - Min Latency - - `gpt-oss-120b-latency.yaml `_ - - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/gpt-oss-120b-latency.yaml`` + - `gpt-oss-120b-latency.yaml `_ + - ``trtllm-serve openai/gpt-oss-120b --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-latency.yaml`` * - `Qwen3-Next-80B-A3B-Thinking `_ - Any - Max Throughput - - `qwen3-next.yaml `_ - - ``trtllm-serve Qwen/Qwen3-Next-80B-A3B-Thinking --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/qwen3-next.yaml`` + - `qwen3-next.yaml `_ + - ``trtllm-serve Qwen/Qwen3-Next-80B-A3B-Thinking --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/qwen3-next.yaml`` * - Qwen3 family (e.g. `Qwen3-30B-A3B `_) - Any - Max Throughput - - `qwen3.yaml `_ - - ``trtllm-serve Qwen/Qwen3-30B-A3B --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/qwen3.yaml`` (swap to another Qwen3 model name as needed) + - `qwen3.yaml `_ + - ``trtllm-serve Qwen/Qwen3-30B-A3B --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/qwen3.yaml`` (swap to another Qwen3 model name as needed) * - `Llama-3.3-70B (FP8) `_ - Any - Max Throughput - - `llama-3.3-70b.yaml `_ - - ``trtllm-serve nvidia/Llama-3.3-70B-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/llama-3.3-70b.yaml`` + - `llama-3.3-70b.yaml `_ + - ``trtllm-serve nvidia/Llama-3.3-70B-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/llama-3.3-70b.yaml`` * - `Llama 4 Scout (FP8) `_ - Any - Max Throughput - - `llama-4-scout.yaml `_ - - ``trtllm-serve nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/llama-4-scout.yaml`` + - `llama-4-scout.yaml `_ + - ``trtllm-serve nvidia/Llama-4-Scout-17B-16E-Instruct-FP8 --extra_llm_api_options ${TRTLLM_DIR}/examples/configs/curated/llama-4-scout.yaml`` Model-Specific Deployment Guides --------------------------------- @@ -105,4 +105,4 @@ Comprehensive Configuration Database The table below lists all available pre-configured model scenarios in the TensorRT LLM configuration database. Each row represents a specific model, GPU, and performance profile combination with recommended request settings. -.. include:: comprehensive_table.rst +.. include:: config_table.rst diff --git a/examples/configs/deepseek-r1-deepgemm.yaml b/examples/configs/curated/deepseek-r1-deepgemm.yaml similarity index 100% rename from examples/configs/deepseek-r1-deepgemm.yaml rename to examples/configs/curated/deepseek-r1-deepgemm.yaml diff --git a/examples/configs/deepseek-r1-latency.yaml b/examples/configs/curated/deepseek-r1-latency.yaml similarity index 100% rename from examples/configs/deepseek-r1-latency.yaml rename to examples/configs/curated/deepseek-r1-latency.yaml diff --git a/examples/configs/deepseek-r1-throughput.yaml b/examples/configs/curated/deepseek-r1-throughput.yaml similarity index 100% rename from examples/configs/deepseek-r1-throughput.yaml rename to examples/configs/curated/deepseek-r1-throughput.yaml diff --git a/examples/configs/gpt-oss-120b-latency.yaml b/examples/configs/curated/gpt-oss-120b-latency.yaml similarity index 100% rename from examples/configs/gpt-oss-120b-latency.yaml rename to examples/configs/curated/gpt-oss-120b-latency.yaml diff --git a/examples/configs/gpt-oss-120b-throughput.yaml b/examples/configs/curated/gpt-oss-120b-throughput.yaml similarity index 100% rename from examples/configs/gpt-oss-120b-throughput.yaml rename to examples/configs/curated/gpt-oss-120b-throughput.yaml diff --git a/examples/configs/llama-3.3-70b.yaml b/examples/configs/curated/llama-3.3-70b.yaml similarity index 100% rename from examples/configs/llama-3.3-70b.yaml rename to examples/configs/curated/llama-3.3-70b.yaml diff --git a/examples/configs/llama-4-scout.yaml b/examples/configs/curated/llama-4-scout.yaml similarity index 100% rename from examples/configs/llama-4-scout.yaml rename to examples/configs/curated/llama-4-scout.yaml diff --git a/examples/configs/qwen3-disagg-prefill.yaml b/examples/configs/curated/qwen3-disagg-prefill.yaml similarity index 100% rename from examples/configs/qwen3-disagg-prefill.yaml rename to examples/configs/curated/qwen3-disagg-prefill.yaml diff --git a/examples/configs/qwen3-next.yaml b/examples/configs/curated/qwen3-next.yaml similarity index 100% rename from examples/configs/qwen3-next.yaml rename to examples/configs/curated/qwen3-next.yaml diff --git a/examples/configs/qwen3.yaml b/examples/configs/curated/qwen3.yaml similarity index 100% rename from examples/configs/qwen3.yaml rename to examples/configs/curated/qwen3.yaml diff --git a/tensorrt_llm/configure/database.py b/examples/configs/database/database.py similarity index 96% rename from tensorrt_llm/configure/database.py rename to examples/configs/database/database.py index a2ef3fba6867..e0c73a8ef1ca 100644 --- a/tensorrt_llm/configure/database.py +++ b/examples/configs/database/database.py @@ -20,7 +20,7 @@ import yaml from pydantic import BaseModel, Field, RootModel -DATABASE_LIST_PATH = Path(__file__).parent / "database" / "scenario_list.yaml" +DATABASE_LIST_PATH = Path(__file__).parent / "lookup.yaml" class RecipeConstraints(BaseModel): diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml b/examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml rename to examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/scenario_list.yaml b/examples/configs/database/lookup.yaml similarity index 53% rename from tensorrt_llm/configure/database/scenario_list.yaml rename to examples/configs/database/lookup.yaml index 031b7846bca2..d1ac7143ceb9 100644 --- a/tensorrt_llm/configure/database/scenario_list.yaml +++ b/examples/configs/database/lookup.yaml @@ -3,1174 +3,1174 @@ isl: 1024 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 128 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 256 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: nvidia/DeepSeek-R1-0528-FP4-v2 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml + config_path: examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: deepseek-ai/DeepSeek-R1-0528 gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml + config_path: examples/configs/database/deepseek-ai/DeepSeek-R1-0528/H200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: B200_NVL isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 1024 osl: 8192 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml num_gpus: 1 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml num_gpus: 2 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml num_gpus: 4 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 16 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 32 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 4 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 64 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml num_gpus: 8 - model: openai/gpt-oss-120b gpu: H200_SXM isl: 8192 osl: 1024 concurrency: 8 - config_path: database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml + config_path: examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml num_gpus: 8 diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc128.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc256.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc128.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc256.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/1k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc128.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc256.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc128.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc256.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml rename to examples/configs/database/nvidia/DeepSeek-R1-0528-FP4-v2/B200/8k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp1_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp2_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp1_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp2_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/1k8k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp1_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp2_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/B200/8k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp1_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp2_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k1k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp1_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp2_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/1k8k_tp8_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp1_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp2_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp4_conc8.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc16.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc32.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc4.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc64.yaml diff --git a/tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml b/examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml similarity index 100% rename from tensorrt_llm/configure/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml rename to examples/configs/database/openai/gpt-oss-120b/H200/8k1k_tp8_conc8.yaml diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 7441f7ab4a18..c3e0a5fb90d4 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -25,7 +25,7 @@ # Load database module directly to avoid tensorrt_llm.__init__ (requires torch) _spec = importlib.util.spec_from_file_location( - "database", REPO_ROOT / "tensorrt_llm" / "configure" / "database.py" + "database", REPO_ROOT / "examples" / "configs" / "database" / "database.py" ) _db = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(_db) @@ -51,7 +51,7 @@ def generate_rst(yaml_path, output_file=None): """Generate RST table from YAML config database. Args: - yaml_path: Path to scenario_list.yaml (str or Path) + yaml_path: Path to lookup.yaml (str or Path) output_file: Optional output file path. If None, prints to stdout. """ recipe_list = RecipeList.from_yaml(Path(yaml_path)) @@ -108,12 +108,10 @@ def generate_rst(yaml_path, output_file=None): for key in sorted_keys: entries = subgroups[key] entries.sort(key=lambda x: x.concurrency) + n = len(entries) + mid = n // 2 - min_conc = entries[0].concurrency - max_conc = entries[-1].concurrency - conc_range = max_conc - min_conc - - for entry in entries: + for idx, entry in enumerate(entries): gpu = entry.gpu num_gpus = entry.num_gpus gpu_display = f"{num_gpus}x{gpu}" if num_gpus and num_gpus > 1 else gpu @@ -122,26 +120,25 @@ def generate_rst(yaml_path, output_file=None): conc = entry.concurrency config_path = entry.config_path - if len(entries) == 1: + if n == 1: if conc <= 16: profile = "Low Latency" elif conc >= 64: profile = "High Throughput" else: profile = "Balanced" + elif idx == 0: + profile = "Min Latency" + elif idx == n - 1: + profile = "Max Throughput" + elif n % 2 == 1 and idx == mid: + profile = "Balanced" + elif idx < mid: + profile = "Low Latency" else: - if conc == min_conc: - profile = "Min Latency" - elif conc == max_conc: - profile = "Max Throughput" - else: - relative_pos = (conc - min_conc) / conc_range - if relative_pos < 0.5: - profile = "Low Latency" - else: - profile = "High Throughput" + profile = "High Throughput" - full_config_path = os.path.join("tensorrt_llm/configure", config_path) + full_config_path = config_path command = f"trtllm-serve {model} --extra_llm_api_options ${{TRTLLM_DIR}}/{full_config_path}" config_filename = os.path.basename(full_config_path) @@ -174,5 +171,5 @@ def generate_rst(yaml_path, output_file=None): if not yaml_path.exists(): print(f"Error: YAML file not found at {yaml_path}", file=sys.stderr) sys.exit(1) - output_path = REPO_ROOT / "docs/source/deployment-guide/comprehensive_table.rst" + output_path = REPO_ROOT / "docs/source/deployment-guide/config_table.rst" generate_rst(yaml_path, output_file=output_path) diff --git a/setup.py b/setup.py index b368bfd082d1..5c61029aad85 100644 --- a/setup.py +++ b/setup.py @@ -132,7 +132,6 @@ def has_ext_modules(self): 'bench/build/benchmark_config.yml', 'evaluate/lm_eval_tasks/**/*', "_torch/auto_deploy/config/*.yaml", - "configure/database/**/*.yaml", # Include CUDA source for fused MoE align extension so runtime JIT can find it in wheels '_torch/auto_deploy/custom_ops/fused_moe/moe_align_kernel.cu', '_torch/auto_deploy/custom_ops/fused_moe/triton_fused_moe_configs/*' diff --git a/tensorrt_llm/configure/__init__.py b/tensorrt_llm/configure/__init__.py deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/tests/unittest/tools/test_generate_config_table.py b/tests/unittest/tools/test_generate_config_table.py index 898169496056..a2dcf66783fa 100644 --- a/tests/unittest/tools/test_generate_config_table.py +++ b/tests/unittest/tools/test_generate_config_table.py @@ -28,7 +28,7 @@ class TestConfigTableSync(unittest.TestCase): def test_config_table_sync(self): - """Test that the comprehensive_table.rst file is synchronized with the scenario_list.yaml database. + """Test that the config_table.rst file is synchronized with the lookup.yaml database. Ensures that the RST file is up-to-date with the YAML database. """ @@ -36,8 +36,8 @@ def test_config_table_sync(self): self.skipTest("generate_config_table not available") # Define paths - yaml_path = os.path.join(REPO_ROOT, "tensorrt_llm/configure/database/scenario_list.yaml") - rst_path = os.path.join(REPO_ROOT, "docs/source/deployment-guide/comprehensive_table.rst") + yaml_path = os.path.join(REPO_ROOT, "examples/configs/database/lookup.yaml") + rst_path = os.path.join(REPO_ROOT, "docs/source/deployment-guide/config_table.rst") # Ensure files exist self.assertTrue(os.path.exists(yaml_path), f"YAML file not found: {yaml_path}") @@ -57,7 +57,7 @@ def test_config_table_sync(self): self.assertEqual( existing_content.strip(), generated_content.strip(), - "comprehensive_table.rst is not synchronized with scenario_list.yaml. " + "config_table.rst is not synchronized with lookup.yaml. " "Please run 'python3 scripts/generate_config_table.py' from the repo root to update it.", ) From 4b37eb7362a9019e2f33d8a89bd83352666effd4 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 08:34:04 -0800 Subject: [PATCH 35/39] reflect path change in other parts of docs Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- .../deployment-guide-for-deepseek-r1-on-trtllm.md | 8 ++++---- .../deployment-guide-for-gpt-oss-on-trtllm.md | 8 ++++---- .../deployment-guide-for-llama3.3-70b-on-trtllm.md | 4 ++-- .../deployment-guide-for-llama4-scout-on-trtllm.md | 4 ++-- .../deployment-guide-for-qwen3-next-on-trtllm.md | 4 ++-- .../deployment-guide-for-qwen3-on-trtllm.md | 4 ++-- examples/models/core/qwen/README.md | 6 +++--- 7 files changed, 19 insertions(+), 19 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md index 07b86fd0ca84..e4165eac09c6 100644 --- a/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-deepseek-r1-on-trtllm.md @@ -66,7 +66,7 @@ We maintain YAML configuration files with recommended performance settings in th ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/deepseek-r1-throughput.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-throughput.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -74,7 +74,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/deepseek-r1-throughput.yaml +```{literalinclude} ../../../examples/configs/curated/deepseek-r1-throughput.yaml --- language: shell prepend: | @@ -90,7 +90,7 @@ To use the `DeepGEMM` MOE backend on B200/GB200, use this config instead: ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/deepseek-r1-deepgemm.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/deepseek-r1-deepgemm.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -98,7 +98,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/deepseek-r1-deepgemm.yaml +```{literalinclude} ../../../examples/configs/curated/deepseek-r1-deepgemm.yaml --- language: shell prepend: | diff --git a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md index 1e181c7ad08a..5a9f9f4c7263 100644 --- a/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-gpt-oss-on-trtllm.md @@ -64,7 +64,7 @@ For low-latency use cases: ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/gpt-oss-120b-latency.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-latency.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -72,7 +72,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/gpt-oss-120b-latency.yaml +```{literalinclude} ../../../examples/configs/curated/gpt-oss-120b-latency.yaml --- language: shell prepend: | @@ -88,7 +88,7 @@ For max-throughput use cases: ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/gpt-oss-120b-throughput.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/gpt-oss-120b-throughput.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -96,7 +96,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/gpt-oss-120b-throughput.yaml +```{literalinclude} ../../../examples/configs/curated/gpt-oss-120b-throughput.yaml --- language: shell prepend: | diff --git a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md index 583ef56b4939..d3e328d810dc 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama3.3-70b-on-trtllm.md @@ -58,7 +58,7 @@ We maintain YAML configuration files with recommended performance settings in th ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/llama-3.3-70b.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/llama-3.3-70b.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -66,7 +66,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/llama-3.3-70b.yaml +```{literalinclude} ../../../examples/configs/curated/llama-3.3-70b.yaml --- language: shell prepend: | diff --git a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md index 10db2e128f39..7d69b7a8be79 100644 --- a/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-llama4-scout-on-trtllm.md @@ -57,7 +57,7 @@ We maintain YAML configuration files with recommended performance settings in th ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/llama-4-scout.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/llama-4-scout.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -65,7 +65,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/llama-4-scout.yaml +```{literalinclude} ../../../examples/configs/curated/llama-4-scout.yaml --- language: shell prepend: | diff --git a/docs/source/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.md index 246fc74a567c..46bf724b7131 100644 --- a/docs/source/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-qwen3-next-on-trtllm.md @@ -35,7 +35,7 @@ We maintain YAML configuration files with recommended performance settings in th ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/qwen3-next.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/qwen3-next.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -43,7 +43,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/qwen3-next.yaml +```{literalinclude} ../../../examples/configs/curated/qwen3-next.yaml --- language: shell prepend: | diff --git a/docs/source/deployment-guide/deployment-guide-for-qwen3-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-qwen3-on-trtllm.md index 190740ebd858..894c6a1e63f1 100644 --- a/docs/source/deployment-guide/deployment-guide-for-qwen3-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-qwen3-on-trtllm.md @@ -40,7 +40,7 @@ We maintain YAML configuration files with recommended performance settings in th ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/qwen3.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/qwen3.yaml ``` Note: if you don't have access to the source code locally, you can manually create the YAML config file using the code in the dropdown below. @@ -48,7 +48,7 @@ Note: if you don't have access to the source code locally, you can manually crea ````{admonition} Show code :class: dropdown -```{literalinclude} ../../../examples/configs/qwen3.yaml +```{literalinclude} ../../../examples/configs/curated/qwen3.yaml --- language: shell prepend: | diff --git a/examples/models/core/qwen/README.md b/examples/models/core/qwen/README.md index 1d3d97b26796..8bdd9e4bf5d5 100644 --- a/examples/models/core/qwen/README.md +++ b/examples/models/core/qwen/README.md @@ -747,7 +747,7 @@ We maintain YAML configuration files with recommended performance settings in th ```shell TRTLLM_DIR=/app/tensorrt_llm # change as needed to match your environment -EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/qwen3.yaml +EXTRA_LLM_API_FILE=${TRTLLM_DIR}/examples/configs/curated/qwen3.yaml ``` #### trtllm-serve @@ -778,7 +778,7 @@ For example, you can launch a single context server on port 8001 with: ```bash export TRTLLM_USE_UCX_KVCACHE=1 export TRTLLM_DIR=/app/tensorrt_llm -export EXTRA_LLM_API_FILE="${TRTLLM_DIR}/examples/configs/qwen3-disagg-prefill.yaml" +export EXTRA_LLM_API_FILE="${TRTLLM_DIR}/examples/configs/curated/qwen3-disagg-prefill.yaml" trtllm-serve Qwen3-30B-A3B/ --port 8001 --extra_llm_api_options ${EXTRA_LLM_API_FILE} &> output_ctx & ``` @@ -788,7 +788,7 @@ And you can launch two generation servers on port 8002 and 8003 with: ```bash export TRTLLM_USE_UCX_KVCACHE=1 export TRTLLM_DIR=/app/tensorrt_llm -export EXTRA_LLM_API_FILE="${TRTLLM_DIR}/examples/configs/qwen3.yaml" +export EXTRA_LLM_API_FILE="${TRTLLM_DIR}/examples/configs/curated/qwen3.yaml" for port in {8002..8003}; do \ trtllm-serve Qwen3-30B-A3B/ --port ${port} --extra_llm_api_options ${EXTRA_LLM_API_FILE} &> output_gen_${port} & \ From 0e52837e0855f8f5fd21f93892a2f66a0db113fe Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:10:00 +0000 Subject: [PATCH 36/39] add llmapi test for configs Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tests/unittest/llmapi/test_config_database.py | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tests/unittest/llmapi/test_config_database.py diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py new file mode 100644 index 000000000000..7314b5441ad6 --- /dev/null +++ b/tests/unittest/llmapi/test_config_database.py @@ -0,0 +1,97 @@ +"""L0 tests for validating config database YAML files against LLM API pydantic models. + +This module validates that all YAML configuration files in examples/configs/database/ +conform to the TorchLlmArgs pydantic model structure. This provides an early +validation check without the cost of actually running trtllm-serve. +""" + +from pathlib import Path +from unittest.mock import Mock, patch + +import pytest +import yaml + +from tensorrt_llm.llmapi.llm_args import TorchLlmArgs, update_llm_args_with_extra_dict + +# Path to config directories +# File is at: tests/unittest/llmapi/test_config_database.py +# parents[0]=llmapi, parents[1]=unittest, parents[2]=tests, parents[3]=tensorrt_llm +CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" +DATABASE_DIR = CONFIG_ROOT / "database" + +# Collect all database config files (excluding lookup.yaml which is an index file) +DATABASE_CONFIGS = ( + [c for c in DATABASE_DIR.rglob("*.yaml") if c.name != "lookup.yaml"] + if DATABASE_DIR.exists() + else [] +) + + +@pytest.fixture(autouse=True) +def mock_gpu_environment(): + """Mock GPU functions for CPU-only test execution. + + The TorchLlmArgs validators include GPU-dependent code: + - validate_gpus_per_node: calls torch.cuda.device_count() + - validate_dtype: calls torch.cuda.get_device_properties(0).major + + This fixture mocks these functions to allow validation tests to run + without requiring GPU access. + """ + mock_props = Mock() + mock_props.major = 8 # Simulate SM80+ GPU (Ampere or newer) + mock_props.name = "Mock GPU" + + with patch("torch.cuda.device_count", return_value=8): + with patch("torch.cuda.get_device_properties", return_value=mock_props): + with patch("torch.cuda.is_available", return_value=True): + yield + + +def get_config_id(config_path: Path) -> str: + """Generate a readable test ID from config path.""" + try: + return str(config_path.relative_to(DATABASE_DIR)) + except ValueError: + return str(config_path.name) + + +@pytest.mark.part0 +@pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) +def test_config_validates_against_llm_args(config_path: Path): + """Validate config YAML against TorchLlmArgs pydantic model. + + This test: + 1. Loads the YAML config file + 2. Creates a base TorchLlmArgs with dummy model + 3. Merges the config using update_llm_args_with_extra_dict + 4. Validates by reconstructing TorchLlmArgs + + Any invalid fields will cause a ValidationError. + """ + # Load config + with open(config_path) as f: + config_dict = yaml.safe_load(f) + + # Handle empty configs + if config_dict is None: + config_dict = {} + + # Create base args with dummy model and skip tokenizer init + # skip_tokenizer_init=True prevents attempting to load a real tokenizer + base_args = TorchLlmArgs(model="dummy/model", skip_tokenizer_init=True) + base_dict = base_args.model_dump() + + # Merge config using the existing update function + merged = update_llm_args_with_extra_dict(base_dict, config_dict) + + # Validate by reconstructing TorchLlmArgs + # This will raise ValidationError if config has invalid fields + TorchLlmArgs(**merged) + + +@pytest.mark.part0 +def test_database_config_count(): + """Sanity check: ensure we found the expected config files.""" + assert len(DATABASE_CONFIGS) > 0, "No database config files found" + print(f"\nFound {len(DATABASE_CONFIGS)} database configs") From ddf96953ec2c104bb7d1ccc4a3f21f654e28dc21 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 18:16:37 +0000 Subject: [PATCH 37/39] add copyright header, deslop Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- tests/unittest/llmapi/test_config_database.py | 73 +++++-------------- 1 file changed, 20 insertions(+), 53 deletions(-) diff --git a/tests/unittest/llmapi/test_config_database.py b/tests/unittest/llmapi/test_config_database.py index 7314b5441ad6..72dfce770f56 100644 --- a/tests/unittest/llmapi/test_config_database.py +++ b/tests/unittest/llmapi/test_config_database.py @@ -1,9 +1,18 @@ -"""L0 tests for validating config database YAML files against LLM API pydantic models. - -This module validates that all YAML configuration files in examples/configs/database/ -conform to the TorchLlmArgs pydantic model structure. This provides an early -validation check without the cost of actually running trtllm-serve. -""" +# SPDX-FileCopyrightText: Copyright (c) 2024-2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""L0 tests for validating config database YAML files against TorchLlmArgs.""" from pathlib import Path from unittest.mock import Mock, patch @@ -13,13 +22,9 @@ from tensorrt_llm.llmapi.llm_args import TorchLlmArgs, update_llm_args_with_extra_dict -# Path to config directories -# File is at: tests/unittest/llmapi/test_config_database.py -# parents[0]=llmapi, parents[1]=unittest, parents[2]=tests, parents[3]=tensorrt_llm CONFIG_ROOT = Path(__file__).parents[3] / "examples" / "configs" DATABASE_DIR = CONFIG_ROOT / "database" -# Collect all database config files (excluding lookup.yaml which is an index file) DATABASE_CONFIGS = ( [c for c in DATABASE_DIR.rglob("*.yaml") if c.name != "lookup.yaml"] if DATABASE_DIR.exists() @@ -29,18 +34,9 @@ @pytest.fixture(autouse=True) def mock_gpu_environment(): - """Mock GPU functions for CPU-only test execution. - - The TorchLlmArgs validators include GPU-dependent code: - - validate_gpus_per_node: calls torch.cuda.device_count() - - validate_dtype: calls torch.cuda.get_device_properties(0).major - - This fixture mocks these functions to allow validation tests to run - without requiring GPU access. - """ + """Mock GPU functions for CPU-only test execution.""" mock_props = Mock() - mock_props.major = 8 # Simulate SM80+ GPU (Ampere or newer) - mock_props.name = "Mock GPU" + mock_props.major = 8 with patch("torch.cuda.device_count", return_value=8): with patch("torch.cuda.get_device_properties", return_value=mock_props): @@ -49,49 +45,20 @@ def mock_gpu_environment(): def get_config_id(config_path: Path) -> str: - """Generate a readable test ID from config path.""" - try: - return str(config_path.relative_to(DATABASE_DIR)) - except ValueError: - return str(config_path.name) + return str(config_path.relative_to(DATABASE_DIR)) @pytest.mark.part0 @pytest.mark.parametrize("config_path", DATABASE_CONFIGS, ids=get_config_id) def test_config_validates_against_llm_args(config_path: Path): - """Validate config YAML against TorchLlmArgs pydantic model. - - This test: - 1. Loads the YAML config file - 2. Creates a base TorchLlmArgs with dummy model - 3. Merges the config using update_llm_args_with_extra_dict - 4. Validates by reconstructing TorchLlmArgs - - Any invalid fields will cause a ValidationError. - """ - # Load config with open(config_path) as f: - config_dict = yaml.safe_load(f) - - # Handle empty configs - if config_dict is None: - config_dict = {} + config_dict = yaml.safe_load(f) or {} - # Create base args with dummy model and skip tokenizer init - # skip_tokenizer_init=True prevents attempting to load a real tokenizer base_args = TorchLlmArgs(model="dummy/model", skip_tokenizer_init=True) - base_dict = base_args.model_dump() - - # Merge config using the existing update function - merged = update_llm_args_with_extra_dict(base_dict, config_dict) - - # Validate by reconstructing TorchLlmArgs - # This will raise ValidationError if config has invalid fields + merged = update_llm_args_with_extra_dict(base_args.model_dump(), config_dict) TorchLlmArgs(**merged) @pytest.mark.part0 def test_database_config_count(): - """Sanity check: ensure we found the expected config files.""" assert len(DATABASE_CONFIGS) > 0, "No database config files found" - print(f"\nFound {len(DATABASE_CONFIGS)} database configs") From 2251feccee484b88d7c751b6771525cf62eedb85 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Mon, 8 Dec 2025 23:14:06 -0800 Subject: [PATCH 38/39] threshold tweak Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- scripts/generate_config_table.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index c3e0a5fb90d4..91c42f56fa61 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -46,6 +46,9 @@ }, } +LOW_LATENCY_CONCURRENCY_THRESHOLD = 8 +HIGH_THROUGHPUT_CONCURRENCY_THRESHOLD = 32 + def generate_rst(yaml_path, output_file=None): """Generate RST table from YAML config database. @@ -109,7 +112,6 @@ def generate_rst(yaml_path, output_file=None): entries = subgroups[key] entries.sort(key=lambda x: x.concurrency) n = len(entries) - mid = n // 2 for idx, entry in enumerate(entries): gpu = entry.gpu @@ -121,9 +123,9 @@ def generate_rst(yaml_path, output_file=None): config_path = entry.config_path if n == 1: - if conc <= 16: + if conc <= LOW_LATENCY_CONCURRENCY_THRESHOLD: profile = "Low Latency" - elif conc >= 64: + elif conc >= HIGH_THROUGHPUT_CONCURRENCY_THRESHOLD: profile = "High Throughput" else: profile = "Balanced" @@ -131,9 +133,9 @@ def generate_rst(yaml_path, output_file=None): profile = "Min Latency" elif idx == n - 1: profile = "Max Throughput" - elif n % 2 == 1 and idx == mid: + elif idx in ((n - 1) // 2, n // 2): profile = "Balanced" - elif idx < mid: + elif idx < n // 2: profile = "Low Latency" else: profile = "High Throughput" From c36cc317ca102bc1c66f5d352712874ca7ee75a0 Mon Sep 17 00:00:00 2001 From: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> Date: Tue, 9 Dec 2025 11:57:56 -0800 Subject: [PATCH 39/39] simplify import Signed-off-by: Venky Ganesh <23023424+venkywonka@users.noreply.github.com> --- scripts/generate_config_table.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/scripts/generate_config_table.py b/scripts/generate_config_table.py index 91c42f56fa61..2d423c0811fc 100644 --- a/scripts/generate_config_table.py +++ b/scripts/generate_config_table.py @@ -14,23 +14,15 @@ # limitations under the License. -import importlib.util import os import sys from collections import defaultdict from pathlib import Path +from examples.configs.database.database import DATABASE_LIST_PATH, RecipeList + SCRIPT_DIR = Path(__file__).parent.resolve() REPO_ROOT = SCRIPT_DIR.parent - -# Load database module directly to avoid tensorrt_llm.__init__ (requires torch) -_spec = importlib.util.spec_from_file_location( - "database", REPO_ROOT / "examples" / "configs" / "database" / "database.py" -) -_db = importlib.util.module_from_spec(_spec) -_spec.loader.exec_module(_db) -DATABASE_LIST_PATH, RecipeList = _db.DATABASE_LIST_PATH, _db.RecipeList - MODEL_INFO = { "deepseek-ai/DeepSeek-R1-0528": { "display_name": "DeepSeek-R1",