From 89a81c8a510603263715e4775ff924e093c14c6b Mon Sep 17 00:00:00 2001 From: Chenfei Zhang Date: Sun, 19 Jul 2026 23:18:42 -0700 Subject: [PATCH] [https://nvbugs/6463987][perf] Revert NGC PyTorch 26.05 stack upgrade (cuBLAS/cuDNN/NCCL) for GB300 GLM-5-fp4 ctx_only NVFP4 regression PR #15087 (commit 02cedf6e4e) upgraded the NGC PyTorch base container 26.04 -> 26.05 and pinned cuBLAS 13.4.0.1 -> 13.4.1.2, cuDNN 9.21.0.82 -> 9.22.0.52, NCCL 2.29.7 -> 2.30.4. On the GB300 ctx_only-gb300_glm-5-fp4_1k1k perf-sanity workload this triggers a ~32% output_token_throughput regression (4.07 -> 2.75 tok/s on oci-aga GB300, 3-rep same-node medians, CV 1.8%). The regression is still present on origin/main HEAD (3fe4ceb9bc, post-merge 2844) measured at 2.58 tok/s (list [2.58, 2.56, 2.63], CV 1.3%) via auto_perf_stabilize's 3-rep reproduce on a clean unpatched wheel. The bug_profile module ranking pins the loss to steady-state prefill GEMMs on the ctx_only NVFP4 workload: cuBLAS 13.4.1.2's NVFP4 heuristic (primary) plus NCCL 2.30 attention-DP allgather changes (tertiary). Startup got FASTER (7m10s -> 5m50s) between good and bad commits which rules out JIT/warmup and pins the loss to steady-state. This diff is a minimal revert of the version pins from PR #15087. No TensorRT-LLM source is touched. Verified on oci-aga GB300 in the auto-perf-fix workflow: 3-rep list [4.15, 4.23, 4.16] median 4.16 tok/s -- clears the (good+bad)/2 threshold 3.41 tok/s by +22% and recovers +61% over current ToT (2.58 tok/s). Changes (infra only, three files): * docker/common/install_tensorrt.sh: cuBLAS 13.4.1.2 -> 13.4.0.1, cuDNN 9.22.0.52 -> 9.21.0.82, NCCL 2.30.4 -> 2.29.7. Aligns comment URLs with the 26.04 release notes. * jenkins/current_image_tags.properties: pins container image tags back to the 26.04 base (202607100544-16206) for x86, sbsa, and both rockylinux variants. * requirements.txt: torch upper bound 2.13.0a0 -> 2.12.0a0, nvidia-nccl-cu13 upper bound 2.30.4 -> 2.29.7. Comment URLs aligned with 26.04. Signed-off-by: Chenfei Zhang --- docker/common/install_tensorrt.sh | 8 ++++---- jenkins/current_image_tags.properties | 10 +++++----- requirements.txt | 8 ++++---- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docker/common/install_tensorrt.sh b/docker/common/install_tensorrt.sh index 0c762e0a3ad3..c53ab622fb45 100644 --- a/docker/common/install_tensorrt.sh +++ b/docker/common/install_tensorrt.sh @@ -4,13 +4,13 @@ set -ex TRT_VER="10.16.1.11" # Align with the pre-installed cuDNN / cuBLAS / NCCL versions from -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-04.html#rel-26-04 CUDA_VER="13.2" # 13.2.1 # Keep the installation for cuDNN if users want to install PyTorch with source codes. # PyTorch 2.x can compile with cuDNN v9. -CUDNN_VER="9.22.0.52-1" -NCCL_VER="2.30.4-1+cuda13.2" -CUBLAS_VER="13.4.1.2-1" +CUDNN_VER="9.21.0.82-1" +NCCL_VER="2.29.7-1+cuda13.2" +CUBLAS_VER="13.4.0.1-1" # Align with the pre-installed CUDA / NVCC / NVRTC versions from # https://docs.nvidia.com/cuda/cuda-toolkit-release-notes/index.html NVRTC_VER="13.2.78-1" diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index 504fd8f234a8..07aa75063b77 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -13,8 +13,8 @@ # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. IMAGE_NAME=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm -LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.05-py3-sbsa-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-trt10.16.1.11-skip-tritondevel-202607151440-16194 -LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.16.1.11-skip-tritondevel-202607151440-16194 +LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-x86_64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607100544-16206 +LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-26.04-py3-aarch64-ubuntu24.04-trt10.16.1.11-skip-tritondevel-202607100544-16206 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py310-trt10.16.1.11-skip-tritondevel-202607100544-16206 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-rocky8-x86_64-rocky8-py312-trt10.16.1.11-skip-tritondevel-202607100544-16206 +LLM_SBSA_WHEEL_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-13.2.1-devel-ubuntu24.04-sbsa-ubuntu24.04-py312-trt10.16.1.11-skip-tritondevel-202607100544-16206 diff --git a/requirements.txt b/requirements.txt index 99d03e56cbc2..42d9ed7b244f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,13 +22,13 @@ pandas h5py==3.12.1 StrEnum sentencepiece>=0.1.99 -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.12.0a0. -torch>=2.11.0,<=2.13.0a0 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-04.html#rel-26-04 uses 2.12.0a0. +torch>=2.11.0,<=2.12.0a0 torchvision nvidia-modelopt[torch]~=0.37.0 -# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-05.html#rel-26-05 uses 2.30.4 +# https://docs.nvidia.com/deeplearning/frameworks/pytorch-release-notes/rel-26-04.html#rel-26-04 uses 2.29.7 # torch 2.11.0+cu130 depends on nvidia-nccl-cu13==2.28.9 -nvidia-nccl-cu13>=2.28.9,<=2.30.4 +nvidia-nccl-cu13>=2.28.9,<=2.29.7 nvidia-cuda-nvrtc transformers==5.5.4 prometheus_client