From 57d525c29476a7919383a2fa1eeaa273c384e81e Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Fri, 8 Aug 2025 00:46:55 +0800 Subject: [PATCH 1/4] [None][infra] Avoid intermittent access broken to nvcr.io Signed-off-by: Yanchao Lu --- docker/Makefile | 10 +++++++--- jenkins/BuildDockerImage.groovy | 11 +++++++++++ jenkins/L0_Test.groovy | 2 +- tests/integration/test_lists/test-db/l0_dgx_h100.yml | 1 + tests/unittest/test_pip_install.py | 3 --- 5 files changed, 20 insertions(+), 7 deletions(-) diff --git a/docker/Makefile b/docker/Makefile index 8432710af43b..b95ea971ef37 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -1,6 +1,8 @@ # Default base image for the docker build as defined in Dockerfile.multi BASE_IMAGE ?= $(shell grep '^ARG BASE_IMAGE=' Dockerfile.multi | grep -o '=.*' | tr -d '="') BASE_TAG ?= $(shell grep '^ARG BASE_TAG=' Dockerfile.multi | grep -o '=.*' | tr -d '="') +TRITON_IMAGE ?= $(shell grep '^ARG TRITON_IMAGE=' Dockerfile.multi | grep -o '=.*' | tr -d '="') +TRITON_BASE_TAG ?= $(shell grep '^ARG TRITON_BASE_TAG=' Dockerfile.multi | grep -o '=.*' | tr -d '="') # Name of the new image IMAGE_NAME ?= tensorrt_llm IMAGE_TAG ?= latest @@ -80,6 +82,8 @@ endef --progress $(DOCKER_PROGRESS) \ $(if $(BASE_IMAGE), --build-arg BASE_IMAGE=$(BASE_IMAGE)) \ $(if $(BASE_TAG), --build-arg BASE_TAG=$(BASE_TAG)) \ + $(if $(TRITON_IMAGE), --build-arg TRITON_IMAGE=$(TRITON_IMAGE)) \ + $(if $(TRITON_BASE_TAG), --build-arg TRITON_BASE_TAG=$(TRITON_BASE_TAG)) \ $(if $(BUILD_WHEEL_ARGS), --build-arg BUILD_WHEEL_ARGS="$(BUILD_WHEEL_ARGS)") \ $(if $(BUILD_WHEEL_SCRIPT), --build-arg BUILD_WHEEL_SCRIPT="$(BUILD_WHEEL_SCRIPT)") \ $(if $(TORCH_INSTALL_TYPE), --build-arg TORCH_INSTALL_TYPE="$(TORCH_INSTALL_TYPE)") \ @@ -187,16 +191,16 @@ jenkins-aarch64_%: STAGE = tritondevel jenkins-rockylinux8_%: PYTHON_VERSION_TAG_ID = $(if $(findstring 3.12,${PYTHON_VERSION}),PY312,$(if $(findstring 3.10,${PYTHON_VERSION}),PY310,$(error Unknown PYTHON_VERSION specified))) jenkins-rockylinux8_%: IMAGE_WITH_TAG = $(shell . ../jenkins/current_image_tags.properties && echo $$LLM_ROCKYLINUX8_${PYTHON_VERSION_TAG_ID}_DOCKER_IMAGE) jenkins-rockylinux8_%: STAGE = tritondevel -jenkins-rockylinux8_%: BASE_IMAGE = nvidia/cuda +jenkins-rockylinux8_%: BASE_IMAGE = nvcr.io/nvidia/cuda jenkins-rockylinux8_%: BASE_TAG = 12.9.1-devel-rockylinux8 rockylinux8_%: STAGE = tritondevel -rockylinux8_%: BASE_IMAGE = nvidia/cuda +rockylinux8_%: BASE_IMAGE = nvcr.io/nvidia/cuda rockylinux8_%: BASE_TAG = 12.9.1-devel-rockylinux8 # For x86_64 and aarch64 ubuntu22_%: STAGE = tritondevel -ubuntu22_%: BASE_IMAGE = nvidia/cuda +ubuntu22_%: BASE_IMAGE = nvcr.io/nvidia/cuda ubuntu22_%: BASE_TAG = 12.9.1-devel-ubuntu22.04 trtllm_%: STAGE = release diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index 5aa61708f5bf..8de703dfcfaa 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -281,9 +281,14 @@ def buildImage(config, imageKeyToTag) try { def build_jobs = BUILD_JOBS // Fix the triton image pull timeout issue + def BASE_IMAGE = sh(script: "cd ${LLM_ROOT} && grep 'ARG BASE_IMAGE=' docker/Dockerfile.multi | grep -o '=.*' | tr -d '=\"'", returnStdout: true).trim() def TRITON_IMAGE = sh(script: "cd ${LLM_ROOT} && grep 'ARG TRITON_IMAGE=' docker/Dockerfile.multi | grep -o '=.*' | tr -d '=\"'", returnStdout: true).trim() def TRITON_BASE_TAG = sh(script: "cd ${LLM_ROOT} && grep 'ARG TRITON_BASE_TAG=' docker/Dockerfile.multi | grep -o '=.*' | tr -d '=\"'", returnStdout: true).trim() + // Replace the base image and triton image with the internal mirror + BASE_IMAGE = BASE_IMAGE.replace("nvcr.io/nvidia/", "urm.nvidia.com/docker/") + TRITON_IMAGE = TRITON_IMAGE.replace("nvcr.io/nvidia/", "urm.nvidia.com/docker/") + if (dependent) { stage ("make ${dependent.target}_${action} (${arch})") { retry(3) { @@ -292,6 +297,8 @@ def buildImage(config, imageKeyToTag) retry(3) { sh """ cd ${LLM_ROOT} && make -C docker ${dependent.target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ TORCH_INSTALL_TYPE=${torchInstallType} \ IMAGE_WITH_TAG=${dependentImageWithTag} \ STAGE=${dependent.dockerfileStage} \ @@ -321,6 +328,8 @@ def buildImage(config, imageKeyToTag) retry(3) { sh """ cd ${LLM_ROOT} && make -C docker ${target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ TORCH_INSTALL_TYPE=${torchInstallType} \ IMAGE_WITH_TAG=${imageWithTag} \ STAGE=${dockerfileStage} \ @@ -336,6 +345,8 @@ def buildImage(config, imageKeyToTag) stage ("custom tag: ${customTag} (${arch})") { sh """ cd ${LLM_ROOT} && make -C docker ${target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ TORCH_INSTALL_TYPE=${torchInstallType} \ IMAGE_WITH_TAG=${customImageWithTag} \ STAGE=${dockerfileStage} \ diff --git a/jenkins/L0_Test.groovy b/jenkins/L0_Test.groovy index b40c7a11a7e4..22a556936066 100644 --- a/jenkins/L0_Test.groovy +++ b/jenkins/L0_Test.groovy @@ -39,7 +39,7 @@ LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE = env.wheelDockerImagePy310 LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE = env.wheelDockerImagePy312 // DLFW torch image -DLFW_IMAGE = "nvcr.io/nvidia/pytorch:25.06-py3" +DLFW_IMAGE = "urm.nvidia.com/docker/nvidia/pytorch:25.06-py3" //Ubuntu base image UBUNTU_22_04_IMAGE = "urm.nvidia.com/docker/ubuntu:22.04" diff --git a/tests/integration/test_lists/test-db/l0_dgx_h100.yml b/tests/integration/test_lists/test-db/l0_dgx_h100.yml index 81369dee61b1..6c39e94f63d2 100644 --- a/tests/integration/test_lists/test-db/l0_dgx_h100.yml +++ b/tests/integration/test_lists/test-db/l0_dgx_h100.yml @@ -189,5 +189,6 @@ l0_dgx_h100: terms: stage: post_merge backend: triton + auto_trigger: others tests: - triton_server/test_triton_llm.py::test_llmapi_backend[4-0-disableDecoupleMode-tensorrt_llm] diff --git a/tests/unittest/test_pip_install.py b/tests/unittest/test_pip_install.py index d75bfbaf4206..11288e09cec0 100644 --- a/tests/unittest/test_pip_install.py +++ b/tests/unittest/test_pip_install.py @@ -51,9 +51,6 @@ def test_pip_install(): help="The wheel path") args = parser.parse_args() - if not os.environ.get("CUDA_HOME"): - os.environ["CUDA_HOME"] = "/usr/local/cuda" - print("########## Install required system libs ##########") if not os.path.exists("/usr/local/mpi/bin/mpicc"): subprocess.check_call("apt-get -y install libopenmpi-dev", shell=True) From 584d0749d3479ee7daae52170dd6d624b380185e Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sun, 10 Aug 2025 10:25:46 +0800 Subject: [PATCH 2/4] Apply suggestion from @ZhanruiSunCh Co-authored-by: Zhanrui Sun <184402041+ZhanruiSunCh@users.noreply.github.com> Signed-off-by: Yanchao Lu --- jenkins/BuildDockerImage.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index 8de703dfcfaa..85e7daeb86aa 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -286,8 +286,8 @@ def buildImage(config, imageKeyToTag) def TRITON_BASE_TAG = sh(script: "cd ${LLM_ROOT} && grep 'ARG TRITON_BASE_TAG=' docker/Dockerfile.multi | grep -o '=.*' | tr -d '=\"'", returnStdout: true).trim() // Replace the base image and triton image with the internal mirror - BASE_IMAGE = BASE_IMAGE.replace("nvcr.io/nvidia/", "urm.nvidia.com/docker/") - TRITON_IMAGE = TRITON_IMAGE.replace("nvcr.io/nvidia/", "urm.nvidia.com/docker/") + BASE_IMAGE = BASE_IMAGE.replace("nvcr.io/", "urm.nvidia.com/docker/") + TRITON_IMAGE = TRITON_IMAGE.replace("nvcr.io/", "urm.nvidia.com/docker/") if (dependent) { stage ("make ${dependent.target}_${action} (${arch})") { From bd376352e944ec0d888cef30846039656d22a319 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Sun, 10 Aug 2025 13:46:33 +0800 Subject: [PATCH 3/4] Update for RockyLinux 8 Signed-off-by: Yanchao Lu --- jenkins/BuildDockerImage.groovy | 52 +++++++++++++++------------------ 1 file changed, 24 insertions(+), 28 deletions(-) diff --git a/jenkins/BuildDockerImage.groovy b/jenkins/BuildDockerImage.groovy index 85e7daeb86aa..fea7d8261c97 100644 --- a/jenkins/BuildDockerImage.groovy +++ b/jenkins/BuildDockerImage.groovy @@ -285,26 +285,26 @@ def buildImage(config, imageKeyToTag) def TRITON_IMAGE = sh(script: "cd ${LLM_ROOT} && grep 'ARG TRITON_IMAGE=' docker/Dockerfile.multi | grep -o '=.*' | tr -d '=\"'", returnStdout: true).trim() def TRITON_BASE_TAG = sh(script: "cd ${LLM_ROOT} && grep 'ARG TRITON_BASE_TAG=' docker/Dockerfile.multi | grep -o '=.*' | tr -d '=\"'", returnStdout: true).trim() + if (target == "rockylinux8") { + BASE_IMAGE = sh(script: "cd ${LLM_ROOT} && grep 'jenkins-rockylinux8_%: BASE_IMAGE =' docker/Makefile | grep -o '=.*' | tr -d '=\"'", returnStdout: true).trim() + } + // Replace the base image and triton image with the internal mirror BASE_IMAGE = BASE_IMAGE.replace("nvcr.io/", "urm.nvidia.com/docker/") TRITON_IMAGE = TRITON_IMAGE.replace("nvcr.io/", "urm.nvidia.com/docker/") if (dependent) { stage ("make ${dependent.target}_${action} (${arch})") { - retry(3) { - sh "docker pull ${TRITON_IMAGE}:${TRITON_BASE_TAG}" - } - retry(3) { - sh """ - cd ${LLM_ROOT} && make -C docker ${dependent.target}_${action} \ - BASE_IMAGE=${BASE_IMAGE} \ - TRITON_IMAGE=${TRITON_IMAGE} \ - TORCH_INSTALL_TYPE=${torchInstallType} \ - IMAGE_WITH_TAG=${dependentImageWithTag} \ - STAGE=${dependent.dockerfileStage} \ - BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} - """ - } + trtllm_utils.llmExecStepWithRetry(this, script: "docker pull ${TRITON_IMAGE}:${TRITON_BASE_TAG}", sleepInSecs: 300, shortCommondRunTimeMax: 7200) + trtllm_utils.llmExecStepWithRetry(this, script: """ + cd ${LLM_ROOT} && make -C docker ${dependent.target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ + TORCH_INSTALL_TYPE=${torchInstallType} \ + IMAGE_WITH_TAG=${dependentImageWithTag} \ + STAGE=${dependent.dockerfileStage} \ + BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} + """, sleepInSecs: 300, shortCommondRunTimeMax: 7200) args += " DEVEL_IMAGE=${dependentImageWithTag}" if (target == "ngc-release") { imageKeyToTag["NGC Devel Image ${config.arch}"] = dependentImageWithTag @@ -322,20 +322,16 @@ def buildImage(config, imageKeyToTag) } } stage ("make ${target}_${action} (${arch})") { - retry(3) { - sh "docker pull ${TRITON_IMAGE}:${TRITON_BASE_TAG}" - } - retry(3) { - sh """ - cd ${LLM_ROOT} && make -C docker ${target}_${action} \ - BASE_IMAGE=${BASE_IMAGE} \ - TRITON_IMAGE=${TRITON_IMAGE} \ - TORCH_INSTALL_TYPE=${torchInstallType} \ - IMAGE_WITH_TAG=${imageWithTag} \ - STAGE=${dockerfileStage} \ - BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} - """ - } + trtllm_utils.llmExecStepWithRetry(this, script: "docker pull ${TRITON_IMAGE}:${TRITON_BASE_TAG}", sleepInSecs: 300, shortCommondRunTimeMax: 7200) + trtllm_utils.llmExecStepWithRetry(this, script: """ + cd ${LLM_ROOT} && make -C docker ${target}_${action} \ + BASE_IMAGE=${BASE_IMAGE} \ + TRITON_IMAGE=${TRITON_IMAGE} \ + TORCH_INSTALL_TYPE=${torchInstallType} \ + IMAGE_WITH_TAG=${imageWithTag} \ + STAGE=${dockerfileStage} \ + BUILD_WHEEL_OPTS='-j ${build_jobs}' ${args} + """, sleepInSecs: 300, shortCommondRunTimeMax: 7200) if (target == "ngc-release") { imageKeyToTag["NGC Release Image ${config.arch}"] = imageWithTag } From f7b0a9e9bc7996126d52b1caf660e709c0f53be1 Mon Sep 17 00:00:00 2001 From: Yanchao Lu Date: Mon, 11 Aug 2025 14:28:14 +0800 Subject: [PATCH 4/4] Update docker image tags Signed-off-by: Yanchao Lu --- jenkins/current_image_tags.properties | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/jenkins/current_image_tags.properties b/jenkins/current_image_tags.properties index dee2ee7218f3..ddf5e2aca9d5 100644 --- a/jenkins/current_image_tags.properties +++ b/jenkins/current_image_tags.properties @@ -11,7 +11,7 @@ # # NB: Typically, the suffix indicates the PR whose CI pipeline generated the images. In case that # images are adopted from PostMerge pipelines, the abbreviated commit hash is used instead. -LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.06-py3-x86_64-ubuntu24.04-trt10.11.0.33-skip-tritondevel-202508051130-6090 -LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.06-py3-aarch64-ubuntu24.04-trt10.11.0.33-skip-tritondevel-202508051130-6090 -LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.1-devel-rocky8-x86_64-rocky8-py310-trt10.11.0.33-skip-tritondevel-202508051130-6090 -LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.1-devel-rocky8-x86_64-rocky8-py312-trt10.11.0.33-skip-tritondevel-202508051130-6090 +LLM_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.06-py3-x86_64-ubuntu24.04-trt10.11.0.33-skip-tritondevel-202508110900-6715 +LLM_SBSA_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:pytorch-25.06-py3-aarch64-ubuntu24.04-trt10.11.0.33-skip-tritondevel-202508110900-6715 +LLM_ROCKYLINUX8_PY310_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.1-devel-rocky8-x86_64-rocky8-py310-trt10.11.0.33-skip-tritondevel-202508110900-6715 +LLM_ROCKYLINUX8_PY312_DOCKER_IMAGE=urm.nvidia.com/sw-tensorrt-docker/tensorrt-llm:cuda-12.9.1-devel-rocky8-x86_64-rocky8-py312-trt10.11.0.33-skip-tritondevel-202508110900-6715