Skip to content

Commit 78ec476

Browse files
vyasrvyasr
authored
Switch to new wheel building pipeline (#3731)
Moves the wheel build and test logic out of the workflow into the repo. This matches conda tests more closely and allows each repo to manage its own wheels more easily. Authors: - Vyas Ramasubramani (https://github.com/vyasr) Approvers: - Ray Douglass (https://github.com/raydouglass) URL: #3731
1 parent f4a9f29 commit 78ec476

File tree

9 files changed

+118
-51
lines changed

9 files changed

+118
-51
lines changed

‎.github/workflows/build.yaml‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -68,27 +68,20 @@ jobs:
6868
sha: ${{ inputs.sha }}
6969
wheel-build-pylibcugraph:
7070
secrets: inherit
71-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
71+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
7272
with:
7373
build_type: ${{ inputs.build_type || 'branch' }}
7474
branch: ${{ inputs.branch }}
7575
sha: ${{ inputs.sha }}
7676
date: ${{ inputs.date }}
77-
package-name: pylibcugraph
78-
package-dir: python/pylibcugraph
79-
80-
# Note that this approach to cloning repos obviates any modification to
81-
# the CMake variables in get_cumlprims_mg.cmake since CMake will just use
82-
# the clone as is.
77+
script: ci/build_wheel_pylibcugraph.sh
8378
extra-repo: rapidsai/cugraph-ops
8479
extra-repo-sha: branch-23.08
8580
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
86-
87-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/"
8881
wheel-publish-pylibcugraph:
8982
needs: wheel-build-pylibcugraph
9083
secrets: inherit
91-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08
84+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08
9285
with:
9386
build_type: ${{ inputs.build_type || 'branch' }}
9487
branch: ${{ inputs.branch }}
@@ -98,28 +91,20 @@ jobs:
9891
wheel-build-cugraph:
9992
needs: wheel-publish-pylibcugraph
10093
secrets: inherit
101-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
94+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
10295
with:
10396
build_type: ${{ inputs.build_type || 'branch' }}
10497
branch: ${{ inputs.branch }}
10598
sha: ${{ inputs.sha }}
10699
date: ${{ inputs.date }}
107-
package-name: cugraph
108-
package-dir: python/cugraph
109-
110-
# Note that this approach to cloning repos obviates any modification to
111-
# the CMake variables in get_cumlprims_mg.cmake since CMake will just use
112-
# the clone as is.
100+
script: ci/build_wheel_cugraph.sh
113101
extra-repo: rapidsai/cugraph-ops
114102
extra-repo-sha: branch-23.08
115103
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
116-
117-
before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse"
118-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/"
119104
wheel-publish-cugraph:
120105
needs: wheel-build-cugraph
121106
secrets: inherit
122-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.08
107+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.08
123108
with:
124109
build_type: ${{ inputs.build_type || 'branch' }}
125110
branch: ${{ inputs.branch }}

‎.github/workflows/pr.yaml‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -78,47 +78,34 @@ jobs:
7878
wheel-build-pylibcugraph:
7979
needs: checks
8080
secrets: inherit
81-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
81+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
8282
with:
8383
build_type: pull-request
84-
package-name: pylibcugraph
85-
package-dir: python/pylibcugraph
84+
script: ci/build_wheel_pylibcugraph.sh
8685
extra-repo: rapidsai/cugraph-ops
8786
extra-repo-sha: branch-23.08
8887
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
89-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/"
9088
wheel-tests-pylibcugraph:
9189
needs: wheel-build-pylibcugraph
9290
secrets: inherit
93-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
91+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
9492
with:
9593
build_type: pull-request
96-
package-name: pylibcugraph
97-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests"
98-
test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py"
94+
script: ci/test_wheel_pylibcugraph.sh
9995
wheel-build-cugraph:
10096
needs: wheel-tests-pylibcugraph
10197
secrets: inherit
102-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.08
98+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.08
10399
with:
104100
build_type: pull-request
105-
package-name: cugraph
106-
package-dir: python/cugraph
101+
script: ci/build_wheel_cugraph.sh
107102
extra-repo: rapidsai/cugraph-ops
108103
extra-repo-sha: branch-23.08
109104
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
110-
before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse"
111-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/"
112105
wheel-tests-cugraph:
113106
needs: wheel-build-cugraph
114107
secrets: inherit
115-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
108+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
116109
with:
117110
build_type: pull-request
118-
package-name: cugraph
119-
# Always want to test against latest dask/distributed.
120-
test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08"
121-
# Skip dataset downloads on arm to save CI time -- arm only runs smoke tests.
122-
test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08"
123-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests"
124-
test-smoketest: "python ci/wheel_smoke_test_cugraph.py"
111+
script: ci/test_wheel_cugraph.sh

‎.github/workflows/test.yaml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,19 @@ jobs:
3232
sha: ${{ inputs.sha }}
3333
wheel-tests-pylibcugraph:
3434
secrets: inherit
35-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
35+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
3636
with:
3737
build_type: nightly
3838
branch: ${{ inputs.branch }}
3939
date: ${{ inputs.date }}
4040
sha: ${{ inputs.sha }}
41-
package-name: pylibcugraph
42-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests"
41+
script: ci/test_wheel_pylibcugraph.sh
4342
wheel-tests-cugraph:
4443
secrets: inherit
45-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.08
44+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.08
4645
with:
4746
build_type: nightly
4847
branch: ${{ inputs.branch }}
4948
date: ${{ inputs.date }}
5049
sha: ${{ inputs.sha }}
51-
package-name: cugraph
52-
# Always want to test against latest dask/distributed.
53-
test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08"
54-
test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08"
55-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests"
50+
script: ci/test_wheel_cugraph.sh

‎ci/build_wheel.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -euo pipefail
5+
6+
package_name=$1
7+
package_dir=$2
8+
9+
source rapids-configure-sccache
10+
source rapids-date-string
11+
12+
# Use gha-tools rapids-pip-wheel-version to generate wheel version then
13+
# update the necessary files
14+
version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})"
15+
16+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
17+
18+
bash ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}"
19+
echo "The package name and/or version was modified in the package source. The git diff is:"
20+
git diff
21+
22+
cd "${package_dir}"
23+
24+
python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
25+
26+
mkdir -p final_dist
27+
python -m auditwheel repair -w final_dist dist/*
28+
29+
RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist

‎ci/build_wheel_cugraph.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -euo pipefail
5+
6+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
7+
8+
RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph
9+
python -m pip install --no-deps ./local-pylibcugraph/pylibcugraph*.whl
10+
11+
export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
12+
13+
./ci/build_wheel.sh cugraph python/cugraph

‎ci/build_wheel_pylibcugraph.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -euo pipefail
5+
6+
export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
7+
8+
./ci/build_wheel.sh pylibcugraph python/pylibcugraph

‎ci/test_wheel.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -eoxu pipefail
5+
6+
package_name=$1
7+
package_dir=$2
8+
9+
mkdir -p ./dist
10+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
11+
12+
# echo to expand wildcard before adding `[extra]` requires for pip
13+
RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
14+
python -m pip install $(echo ./dist/${package_name}*.whl)[test]
15+
16+
# Run smoke tests for aarch64 pull requests
17+
arch=$(uname -m)
18+
if [[ "${arch}" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
19+
python ./ci/wheel_smoke_test_${package_name}.py
20+
else
21+
RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets python -m pytest ./python/${package_name}/${package_name}/tests
22+
fi

‎ci/test_wheel_cugraph.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -eoxu pipefail
5+
6+
# Download the pylibcugraph built in the previous step
7+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
8+
RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep
9+
python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl
10+
11+
# Always install latest dask for testing
12+
python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.08
13+
14+
# Only download test data for x86
15+
arch=$(uname -m)
16+
if [[ "${arch}" == "x86_64" ]]; then
17+
pushd ./datasets
18+
bash ./get_test_data.sh
19+
popd
20+
fi
21+
22+
./ci/test_wheel.sh cugraph python/cugraph

‎ci/test_wheel_pylibcugraph.sh‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -eoxu pipefail
5+
6+
./ci/test_wheel.sh pylibcugraph python/pylibcugraph

0 commit comments

Comments
 (0)