From ac9b67668ae7f6575221cd0327834ba3a27bdb40 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Wed, 26 Nov 2025 07:51:50 +0000 Subject: [PATCH 1/6] Install cuda-compat-13-0 for bare-metal ubuntu 2204 and 2404 Signed-off-by: Yiqing Yan --- docs/source/installation/linux.md | 6 ++++-- jenkins/L0_Test.groovy | 1 + 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/docs/source/installation/linux.md b/docs/source/installation/linux.md index f2a37297ef10..7758a295dd32 100644 --- a/docs/source/installation/linux.md +++ b/docs/source/installation/linux.md @@ -9,8 +9,10 @@ Before the pre-built Python wheel can be installed via `pip`, a few prerequisites must be put into place: - Install CUDA Toolkit following the [CUDA Installation Guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/) and - make sure `CUDA_HOME` environment variable is properly set. + Install CUDA Toolkit 13.0 following the [CUDA Installation Guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/) + and make sure `CUDA_HOME` environment variable is properly set. + `cuda-compat-13-0` package may be required depending on your system's NVIDIA GPU + driver version. For additional information, refer to [CUDA Forward Compatibility](https://docs.nvidia.com/deploy/cuda-compatibility/forward-compatibility.html). ```bash # By default, PyTorch CUDA 12.8 package is installed. Install PyTorch CUDA 13.0 package to align with the CUDA version used for building TensorRT LLM wheels. diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index c286989e4d29..3cc3ed6826c3 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2359,6 +2359,7 @@ def launchTestJobs(pipeline, testFilter) trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget https://developer.download.nvidia.com/compute/cuda/repos/${ubuntu_version}/${platform}/cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "dpkg -i cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-toolkit-13-0") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install -y cuda-compat-13-0 && export LD_LIBRARY_PATH=/usr/local/cuda/compat:$LD_LIBRARY_PATH") } // Extra PyTorch CUDA 13.0 install for all bare-metal environments (Default PyTorch is for CUDA 12.8) if (values[6]) { From 1fb41e46f04cf9931489a8e1e29bef0705da4b23 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Thu, 27 Nov 2025 06:37:15 +0000 Subject: [PATCH 2/6] fix Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 3cc3ed6826c3..d809897a01da 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2359,7 +2359,7 @@ def launchTestJobs(pipeline, testFilter) trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget https://developer.download.nvidia.com/compute/cuda/repos/${ubuntu_version}/${platform}/cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "dpkg -i cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-toolkit-13-0") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install -y cuda-compat-13-0 && export LD_LIBRARY_PATH=/usr/local/cuda/compat:$LD_LIBRARY_PATH") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install -y cuda-compat-13-0 && export LD_LIBRARY_PATH=/usr/local/cuda/compat:\$LD_LIBRARY_PATH") } // Extra PyTorch CUDA 13.0 install for all bare-metal environments (Default PyTorch is for CUDA 12.8) if (values[6]) { From 79ec68edd84f8854f07403bc1369e7b5ea3f1e15 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Fri, 28 Nov 2025 15:34:11 +0800 Subject: [PATCH 3/6] Update docs/source/installation/linux.md Co-authored-by: Yanchao Lu Signed-off-by: Yiqing Yan --- docs/source/installation/linux.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/source/installation/linux.md b/docs/source/installation/linux.md index 7758a295dd32..4e27f0967bb9 100644 --- a/docs/source/installation/linux.md +++ b/docs/source/installation/linux.md @@ -11,8 +11,9 @@ Install CUDA Toolkit 13.0 following the [CUDA Installation Guide for Linux](https://docs.nvidia.com/cuda/cuda-installation-guide-linux/) and make sure `CUDA_HOME` environment variable is properly set. - `cuda-compat-13-0` package may be required depending on your system's NVIDIA GPU - driver version. For additional information, refer to [CUDA Forward Compatibility](https://docs.nvidia.com/deploy/cuda-compatibility/forward-compatibility.html). + + The `cuda-compat-13-0` package may be required depending on your system's NVIDIA GPU + driver version. For additional information, refer to the [CUDA Forward Compatibility](https://docs.nvidia.com/deploy/cuda-compatibility/forward-compatibility.html). ```bash # By default, PyTorch CUDA 12.8 package is installed. Install PyTorch CUDA 13.0 package to align with the CUDA version used for building TensorRT LLM wheels. From c7dca1007ec6f9f374ba65abd5156f6c2541d873 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Fri, 28 Nov 2025 07:46:27 +0000 Subject: [PATCH 4/6] fix LD_LIBRARY_PATH Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index d809897a01da..d2bfb20550be 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2359,7 +2359,7 @@ def launchTestJobs(pipeline, testFilter) trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget https://developer.download.nvidia.com/compute/cuda/repos/${ubuntu_version}/${platform}/cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "dpkg -i cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-toolkit-13-0") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install -y cuda-compat-13-0 && export LD_LIBRARY_PATH=/usr/local/cuda/compat:\$LD_LIBRARY_PATH") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install -y cuda-compat-13-0") } // Extra PyTorch CUDA 13.0 install for all bare-metal environments (Default PyTorch is for CUDA 12.8) if (values[6]) { @@ -2368,6 +2368,9 @@ def launchTestJobs(pipeline, testFilter) } def libEnv = [] + if (values[5] != DLFW_IMAGE) { + libEnv += ["LD_LIBRARY_PATH+compat=/usr/local/cuda/compat"] + } if (env.alternativeTRT) { stage("Replace TensorRT") { trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver) From fc43fe8297e896e73598b833d5429388b17ee080 Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Fri, 28 Nov 2025 08:06:24 +0000 Subject: [PATCH 5/6] review fix Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index d2bfb20550be..73023c2d1fc3 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2359,7 +2359,7 @@ def launchTestJobs(pipeline, testFilter) trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget https://developer.download.nvidia.com/compute/cuda/repos/${ubuntu_version}/${platform}/cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "dpkg -i cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-toolkit-13-0") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get install -y cuda-compat-13-0") + trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-compat-13-0") } // Extra PyTorch CUDA 13.0 install for all bare-metal environments (Default PyTorch is for CUDA 12.8) if (values[6]) { From 90dac63ec2969ceeb8a8f47d15e81e5e62c993cd Mon Sep 17 00:00:00 2001 From: Yiqing Yan Date: Sat, 29 Nov 2025 14:27:31 +0000 Subject: [PATCH 6/6] remove cuda compat Signed-off-by: Yiqing Yan --- jenkins/L0_Test.groovy | 4 ---- 1 file changed, 4 deletions(-) diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index 73023c2d1fc3..c286989e4d29 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -2359,7 +2359,6 @@ def launchTestJobs(pipeline, testFilter) trtllm_utils.llmExecStepWithRetry(pipeline, script: "wget https://developer.download.nvidia.com/compute/cuda/repos/${ubuntu_version}/${platform}/cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "dpkg -i cuda-keyring_1.1-1_all.deb") trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-toolkit-13-0") - trtllm_utils.llmExecStepWithRetry(pipeline, script: "apt-get update && apt-get install -y cuda-compat-13-0") } // Extra PyTorch CUDA 13.0 install for all bare-metal environments (Default PyTorch is for CUDA 12.8) if (values[6]) { @@ -2368,9 +2367,6 @@ def launchTestJobs(pipeline, testFilter) } def libEnv = [] - if (values[5] != DLFW_IMAGE) { - libEnv += ["LD_LIBRARY_PATH+compat=/usr/local/cuda/compat"] - } if (env.alternativeTRT) { stage("Replace TensorRT") { trtllm_utils.replaceWithAlternativeTRT(env.alternativeTRT, cpver)