From 46d28d6797953138157a75c94bc1cd3816bf7be6 Mon Sep 17 00:00:00 2001 From: yukirora Date: Tue, 23 Jul 2024 06:50:09 +0000 Subject: [PATCH 1/7] Fix msccl build error in pipeline --- .github/workflows/build-image.yml | 2 +- third_party/Makefile | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 952430d56..81ce73e25 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -29,7 +29,7 @@ jobs: dockerfile: cuda12.4 tags: superbench/main:cuda12.4 runner: [self-hosted, rocm-build] - build_args: "NUM_MAKE_JOBS=32" + build_args: "NUM_MAKE_JOBS=1" - name: cuda12.2 dockerfile: cuda12.2 tags: superbench/main:cuda12.2 diff --git a/third_party/Makefile b/third_party/Makefile index 69623af8d..fa41370b0 100755 --- a/third_party/Makefile +++ b/third_party/Makefile @@ -14,6 +14,8 @@ ROCBLAS_BRANCH ?= rocm-$(shell dpkg -l | grep 'rocm-dev ' | awk '{print $$3}' | HIPBLASLT_BRANCH ?= rocm-$(shell dpkg -l | grep 'rocm-dev ' | awk '{print $$3}' | cut -d '.' -f1-3) ROCM_VER ?= $(shell hipconfig -R | grep -oP '\d+\.\d+\.\d+' || echo "0.0.0") +NUM_MAKE_JOBS ?= $(shell nproc --ignore=2) + .PHONY: all cuda_with_msccl cuda rocm common cuda_cutlass cuda_bandwidthTest cuda_nccl_tests cuda_perftest cuda_msccl rocm_perftest fio rocm_rccl_tests rocm_rocblas rocm_bandwidthTest gpcnet cuda_gpuburn cpu_stream cpu_hpl directx_amf_encoding_latency directx_amd rocm_hipblaslt megatron_lm megatron_deepspeed apex_rocm # Build all targets. @@ -214,21 +216,21 @@ apex_rocm: cuda_msccl: sb_micro_path ifneq (,$(wildcard msccl/executor/msccl-executor-nccl/Makefile)) cd ./msccl/executor/msccl-executor-nccl && \ - make -j $(shell nproc --ignore=2) src.build && \ + make -j ${NUM_MAKE_JOBS} src.build && \ cd ../../.. mkdir -p $(SB_MICRO_PATH)/lib/msccl-executor-nccl && \ cp -r -v ./msccl/executor/msccl-executor-nccl/build/* $(SB_MICRO_PATH)/lib/msccl-executor-nccl/ endif ifneq (,$(wildcard msccl/scheduler/msccl-scheduler/Makefile)) cd ./msccl/scheduler/msccl-scheduler && \ - CXX=nvcc BIN_HOME=$(SB_MICRO_PATH)/lib/msccl-executor-nccl SRC_HOME=../../../msccl/executor/msccl-executor-nccl make -j $(shell nproc --ignore=2) && \ + CXX=nvcc BIN_HOME=$(SB_MICRO_PATH)/lib/msccl-executor-nccl SRC_HOME=../../../msccl/executor/msccl-executor-nccl make -j ${NUM_MAKE_JOBS} && \ cd ../../.. mkdir -p $(SB_MICRO_PATH)/lib/msccl-scheduler && \ cp -r -v ./msccl/scheduler/msccl-scheduler/build/* $(SB_MICRO_PATH)/lib/msccl-scheduler/ endif ifneq (,$(wildcard msccl/tests/msccl-tests-nccl/Makefile)) cd ./msccl/tests/msccl-tests-nccl && \ - make MPI=1 MPI_HOME=$(MPI_HOME) NCCL_HOME=$(SB_MICRO_PATH)/lib/msccl-executor-nccl -j $(shell nproc --ignore=2) && cd ../../.. + make MPI=1 MPI_HOME=$(MPI_HOME) NCCL_HOME=$(SB_MICRO_PATH)/lib/msccl-executor-nccl -j ${NUM_MAKE_JOBS} && cd ../../.. mkdir -p $(SB_MICRO_PATH)/bin/msccl-tests-nccl && \ cp -r -v ./msccl/tests/msccl-tests-nccl/build/* $(SB_MICRO_PATH)/bin/msccl-tests-nccl/ endif From e596210ed7deab6e9a75d3e878cd4e50e37389cf Mon Sep 17 00:00:00 2001 From: Yuting Jiang Date: Tue, 23 Jul 2024 18:20:36 +0800 Subject: [PATCH 2/7] Update build-image.yml --- .github/workflows/build-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 81ce73e25..0fad064fe 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -29,12 +29,12 @@ jobs: dockerfile: cuda12.4 tags: superbench/main:cuda12.4 runner: [self-hosted, rocm-build] - build_args: "NUM_MAKE_JOBS=1" + build_args: "NUM_MAKE_JOBS=8" - name: cuda12.2 dockerfile: cuda12.2 tags: superbench/main:cuda12.2 runner: [self-hosted, rocm-build] - build_args: "NUM_MAKE_JOBS=64" + build_args: "NUM_MAKE_JOBS=8" - name: cuda11.1.1 dockerfile: cuda11.1.1 tags: superbench/main:cuda11.1.1,superbench/superbench:latest From 5c3770fa4bd13fe0e61f238dae012f5cbdadd626 Mon Sep 17 00:00:00 2001 From: Yuting Jiang Date: Wed, 24 Jul 2024 09:03:09 +0800 Subject: [PATCH 3/7] Update build-image.yml --- .github/workflows/build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 0fad064fe..b5d256f83 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -29,7 +29,7 @@ jobs: dockerfile: cuda12.4 tags: superbench/main:cuda12.4 runner: [self-hosted, rocm-build] - build_args: "NUM_MAKE_JOBS=8" + build_args: "NUM_MAKE_JOBS=4" - name: cuda12.2 dockerfile: cuda12.2 tags: superbench/main:cuda12.2 From 8e3254f554831218cf012fb81e2ad1d56feca8a4 Mon Sep 17 00:00:00 2001 From: Yuting Jiang Date: Thu, 25 Jul 2024 08:57:01 +0800 Subject: [PATCH 4/7] update rocm dockerfile to bulild rccl with release tag due to rccl:msccl build error --- dockerfile/rocm5.7.x.dockerfile | 2 +- dockerfile/rocm6.0.x.dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dockerfile/rocm5.7.x.dockerfile b/dockerfile/rocm5.7.x.dockerfile index 87af0e368..6599c0ca6 100644 --- a/dockerfile/rocm5.7.x.dockerfile +++ b/dockerfile/rocm5.7.x.dockerfile @@ -133,7 +133,7 @@ RUN cd /tmp && \ # Install RCCL RUN cd /opt/ && \ - git clone https://github.com/ROCmSoftwarePlatform/rccl.git && \ + git clone -b release/rocm-rel-5.7 https://github.com/ROCmSoftwarePlatform/rccl.git && \ cd rccl && \ mkdir build && \ cd build && \ diff --git a/dockerfile/rocm6.0.x.dockerfile b/dockerfile/rocm6.0.x.dockerfile index 648bc195d..bd33e289a 100644 --- a/dockerfile/rocm6.0.x.dockerfile +++ b/dockerfile/rocm6.0.x.dockerfile @@ -139,7 +139,7 @@ RUN cd /tmp && \ # Install RCCL RUN cd /opt/ && \ - git clone https://github.com/ROCmSoftwarePlatform/rccl.git && \ + git clone -b release/rocm-rel-6.0 https://github.com/ROCmSoftwarePlatform/rccl.git && \ cd rccl && \ mkdir build && \ cd build && \ From 656506a7872632ef076c13f34f30dd2b47b0fac3 Mon Sep 17 00:00:00 2001 From: Yuting Jiang Date: Thu, 25 Jul 2024 21:59:04 +0800 Subject: [PATCH 5/7] fix rocm5.7 rocblas build error --- dockerfile/rocm5.7.x.dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/dockerfile/rocm5.7.x.dockerfile b/dockerfile/rocm5.7.x.dockerfile index 6599c0ca6..85ba1919e 100644 --- a/dockerfile/rocm5.7.x.dockerfile +++ b/dockerfile/rocm5.7.x.dockerfile @@ -167,6 +167,17 @@ ADD third_party third_party # Apply patch RUN cd third_party/perftest && \ git apply ../perftest_rocm6.patch + +# Update package index and install dependencies of ROCBLAS +RUN apt-get update && \ + apt-get install -y software-properties-common && \ + add-apt-repository universe && \ + apt-get update && \ + apt-get install -y --fix-missing \ + make python3 python3-yaml python3-venv python3-joblib 'python3*-pip' libmsgpack-dev gfortran libomp-dev && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + RUN make RCCL_HOME=/opt/rccl/build/ ROCBLAS_BRANCH=release/rocm-rel-5.7.1.1 HIPBLASLT_BRANCH=release/rocm-rel-5.7 ROCM_VER=rocm-5.5.0 -C third_party rocm -o cpu_hpl -o cpu_stream -o megatron_lm ADD . . From 0adb33c7495447886e042179840bcbe799e50622 Mon Sep 17 00:00:00 2001 From: Yuting Jiang Date: Sat, 27 Jul 2024 15:32:47 +0800 Subject: [PATCH 6/7] Update build-image.yml --- .github/workflows/build-image.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index b5d256f83..22471074a 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -29,7 +29,7 @@ jobs: dockerfile: cuda12.4 tags: superbench/main:cuda12.4 runner: [self-hosted, rocm-build] - build_args: "NUM_MAKE_JOBS=4" + build_args: "NUM_MAKE_JOBS=1" - name: cuda12.2 dockerfile: cuda12.2 tags: superbench/main:cuda12.2 From 066581dd3aba2f528d666a0fe28e6c354d89ccd8 Mon Sep 17 00:00:00 2001 From: Yuting Jiang Date: Sun, 28 Jul 2024 21:33:00 +0800 Subject: [PATCH 7/7] Update build-image.yml --- .github/workflows/build-image.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-image.yml b/.github/workflows/build-image.yml index 22471074a..78efa7800 100644 --- a/.github/workflows/build-image.yml +++ b/.github/workflows/build-image.yml @@ -18,7 +18,7 @@ jobs: docker: name: Docker build ${{ matrix.name }} runs-on: ${{ matrix.runner }} - timeout-minutes: 600 + timeout-minutes: 1200 permissions: contents: read packages: write @@ -29,7 +29,7 @@ jobs: dockerfile: cuda12.4 tags: superbench/main:cuda12.4 runner: [self-hosted, rocm-build] - build_args: "NUM_MAKE_JOBS=1" + build_args: "NUM_MAKE_JOBS=8" - name: cuda12.2 dockerfile: cuda12.2 tags: superbench/main:cuda12.2