Skip to content

Commit 04902cf

Browse files
vyasrvyasr
committed
Merge branch 'branch-23.08' into branch-23.10-merge-23.08
2 parents 41fdb5f + e68d1dd commit 04902cf

File tree

91 files changed

+4066
-1108
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

91 files changed

+4066
-1108
lines changed

‎.github/workflows/build.yaml‎

Lines changed: 6 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -68,27 +68,20 @@ jobs:
6868
sha: ${{ inputs.sha }}
6969
wheel-build-pylibcugraph:
7070
secrets: inherit
71-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.10
71+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10
7272
with:
7373
build_type: ${{ inputs.build_type || 'branch' }}
7474
branch: ${{ inputs.branch }}
7575
sha: ${{ inputs.sha }}
7676
date: ${{ inputs.date }}
77-
package-name: pylibcugraph
78-
package-dir: python/pylibcugraph
79-
80-
# Note that this approach to cloning repos obviates any modification to
81-
# the CMake variables in get_cumlprims_mg.cmake since CMake will just use
82-
# the clone as is.
77+
script: ci/build_wheel_pylibcugraph.sh
8378
extra-repo: rapidsai/cugraph-ops
8479
extra-repo-sha: branch-23.10
8580
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
86-
87-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/"
8881
wheel-publish-pylibcugraph:
8982
needs: wheel-build-pylibcugraph
9083
secrets: inherit
91-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.10
84+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.10
9285
with:
9386
build_type: ${{ inputs.build_type || 'branch' }}
9487
branch: ${{ inputs.branch }}
@@ -98,28 +91,20 @@ jobs:
9891
wheel-build-cugraph:
9992
needs: wheel-publish-pylibcugraph
10093
secrets: inherit
101-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.10
94+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10
10295
with:
10396
build_type: ${{ inputs.build_type || 'branch' }}
10497
branch: ${{ inputs.branch }}
10598
sha: ${{ inputs.sha }}
10699
date: ${{ inputs.date }}
107-
package-name: cugraph
108-
package-dir: python/cugraph
109-
110-
# Note that this approach to cloning repos obviates any modification to
111-
# the CMake variables in get_cumlprims_mg.cmake since CMake will just use
112-
# the clone as is.
100+
script: ci/build_wheel_cugraph.sh
113101
extra-repo: rapidsai/cugraph-ops
114102
extra-repo-sha: branch-23.10
115103
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
116-
117-
before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse"
118-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/"
119104
wheel-publish-cugraph:
120105
needs: wheel-build-cugraph
121106
secrets: inherit
122-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-publish.yml@branch-23.10
107+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-publish.yaml@branch-23.10
123108
with:
124109
build_type: ${{ inputs.build_type || 'branch' }}
125110
branch: ${{ inputs.branch }}

‎.github/workflows/pr.yaml‎

Lines changed: 8 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -78,47 +78,34 @@ jobs:
7878
wheel-build-pylibcugraph:
7979
needs: checks
8080
secrets: inherit
81-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.10
81+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10
8282
with:
8383
build_type: pull-request
84-
package-name: pylibcugraph
85-
package-dir: python/pylibcugraph
84+
script: ci/build_wheel_pylibcugraph.sh
8685
extra-repo: rapidsai/cugraph-ops
8786
extra-repo-sha: branch-23.10
8887
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
89-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/pylibcugraph/cugraph-ops/"
9088
wheel-tests-pylibcugraph:
9189
needs: wheel-build-pylibcugraph
9290
secrets: inherit
93-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.10
91+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10
9492
with:
9593
build_type: pull-request
96-
package-name: pylibcugraph
97-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests"
98-
test-smoketest: "python ci/wheel_smoke_test_pylibcugraph.py"
94+
script: ci/test_wheel_pylibcugraph.sh
9995
wheel-build-cugraph:
10096
needs: wheel-tests-pylibcugraph
10197
secrets: inherit
102-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-build.yml@branch-23.10
98+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-build.yaml@branch-23.10
10399
with:
104100
build_type: pull-request
105-
package-name: cugraph
106-
package-dir: python/cugraph
101+
script: ci/build_wheel_cugraph.sh
107102
extra-repo: rapidsai/cugraph-ops
108103
extra-repo-sha: branch-23.10
109104
extra-repo-deploy-key: CUGRAPH_OPS_SSH_PRIVATE_DEPLOY_KEY
110-
before-wheel: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 /local-wheelhouse"
111-
skbuild-configure-options: "-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/python/cugraph/cugraph-ops/"
112105
wheel-tests-cugraph:
113106
needs: wheel-build-cugraph
114107
secrets: inherit
115-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.10
108+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10
116109
with:
117110
build_type: pull-request
118-
package-name: cugraph
119-
# Always want to test against latest dask/distributed.
120-
test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.10"
121-
# Skip dataset downloads on arm to save CI time -- arm only runs smoke tests.
122-
test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_${{ '${PIP_CU_VERSION}' }} rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.10"
123-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests"
124-
test-smoketest: "python ci/wheel_smoke_test_cugraph.py"
111+
script: ci/test_wheel_cugraph.sh

‎.github/workflows/test.yaml‎

Lines changed: 4 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -32,24 +32,19 @@ jobs:
3232
sha: ${{ inputs.sha }}
3333
wheel-tests-pylibcugraph:
3434
secrets: inherit
35-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.10
35+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10
3636
with:
3737
build_type: nightly
3838
branch: ${{ inputs.branch }}
3939
date: ${{ inputs.date }}
4040
sha: ${{ inputs.sha }}
41-
package-name: pylibcugraph
42-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=./datasets python -m pytest ./python/pylibcugraph/pylibcugraph/tests"
41+
script: ci/test_wheel_pylibcugraph.sh
4342
wheel-tests-cugraph:
4443
secrets: inherit
45-
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-manylinux-test.yml@branch-23.10
44+
uses: rapidsai/shared-action-workflows/.github/workflows/wheels-test.yaml@branch-23.10
4645
with:
4746
build_type: nightly
4847
branch: ${{ inputs.branch }}
4948
date: ${{ inputs.date }}
5049
sha: ${{ inputs.sha }}
51-
package-name: cugraph
52-
# Always want to test against latest dask/distributed.
53-
test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.10"
54-
test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.10"
55-
test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets python -m pytest -m sg ./python/cugraph/cugraph/tests"
50+
script: ci/test_wheel_cugraph.sh

‎ci/build_wheel.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -euo pipefail
5+
6+
package_name=$1
7+
package_dir=$2
8+
9+
source rapids-configure-sccache
10+
source rapids-date-string
11+
12+
# Use gha-tools rapids-pip-wheel-version to generate wheel version then
13+
# update the necessary files
14+
version_override="$(rapids-pip-wheel-version ${RAPIDS_DATE_STRING})"
15+
16+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
17+
18+
bash ci/release/apply_wheel_modifications.sh ${version_override} "-${RAPIDS_PY_CUDA_SUFFIX}"
19+
echo "The package name and/or version was modified in the package source. The git diff is:"
20+
git diff
21+
22+
cd "${package_dir}"
23+
24+
python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
25+
26+
mkdir -p final_dist
27+
python -m auditwheel repair -w final_dist dist/*
28+
29+
RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist

‎ci/build_wheel_cugraph.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -euo pipefail
5+
6+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
7+
8+
RAPIDS_PY_WHEEL_NAME=pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX} rapids-download-wheels-from-s3 ./local-pylibcugraph
9+
python -m pip install --no-deps ./local-pylibcugraph/pylibcugraph*.whl
10+
11+
export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
12+
13+
./ci/build_wheel.sh cugraph python/cugraph

‎ci/build_wheel_pylibcugraph.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -euo pipefail
5+
6+
export SKBUILD_CONFIGURE_OPTIONS="-DDETECT_CONDA_ENV=OFF -DCUGRAPH_BUILD_WHEELS=ON -DFIND_CUGRAPH_CPP=OFF -DCPM_cugraph-ops_SOURCE=${GITHUB_WORKSPACE}/cugraph-ops/"
7+
8+
./ci/build_wheel.sh pylibcugraph python/pylibcugraph

‎ci/release/update-version.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,8 @@ sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-ser
7070
sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/cugraph-service/server/pyproject.toml
7171
sed_runner "s/^version = .*/version = \"${NEXT_FULL_TAG}\"/g" python/pylibcugraph/pyproject.toml
7272

73+
sed_runner "s/branch-.*/branch-${NEXT_SHORT_TAG}/g" ci/test_wheel_cugraph.sh
74+
7375
# Need to distutils-normalize the original version
7476
NEXT_SHORT_TAG_PEP440=$(python -c "from setuptools.extern import packaging; print(packaging.version.Version('${NEXT_SHORT_TAG}'))")
7577

‎ci/test_cpp.sh‎

Lines changed: 9 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -41,19 +41,17 @@ EXITCODE=0
4141
trap "EXITCODE=1" ERR
4242
set +e
4343

44+
export GTEST_OUTPUT=xml:${RAPIDS_TESTS_DIR}/
45+
4446
# Run libcugraph gtests from libcugraph-tests package
4547
rapids-logger "Run gtests"
46-
for gt in "$CONDA_PREFIX"/bin/gtests/libcugraph/* ; do
47-
test_name=$(basename ${gt})
48-
echo "Running gtest $test_name"
49-
${gt} --gtest_output=xml:${RAPIDS_TESTS_DIR}
50-
done
51-
52-
for ct in "$CONDA_PREFIX"/bin/gtests/libcugraph_c/CAPI_*_TEST ; do
53-
test_name=$(basename ${ct})
54-
echo "Running C API test $test_name"
55-
${ct}
56-
done
48+
cd "$CONDA_PREFIX"/bin/gtests/libcugraph/
49+
ctest -j10 --output-on-failure
50+
51+
if [ -d "$CONDA_PREFIX"/bin/gtests/libcugraph_c/ ]; then
52+
cd "$CONDA_PREFIX"/bin/gtests/libcugraph_c/
53+
ctest -j10 --output-on-failure
54+
fi
5755

5856
rapids-logger "Test script exiting with value: $EXITCODE"
5957
exit ${EXITCODE}

‎ci/test_wheel.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -eoxu pipefail
5+
6+
package_name=$1
7+
package_dir=$2
8+
9+
mkdir -p ./dist
10+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
11+
12+
# echo to expand wildcard before adding `[extra]` requires for pip
13+
RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
14+
python -m pip install $(echo ./dist/${package_name}*.whl)[test]
15+
16+
# Run smoke tests for aarch64 pull requests
17+
arch=$(uname -m)
18+
if [[ "${arch}" == "aarch64" && ${RAPIDS_BUILD_TYPE} == "pull-request" ]]; then
19+
python ./ci/wheel_smoke_test_${package_name}.py
20+
else
21+
RAPIDS_DATASET_ROOT_DIR=`pwd`/datasets python -m pytest ./python/${package_name}/${package_name}/tests
22+
fi

‎ci/test_wheel_cugraph.sh‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/bin/bash
2+
# Copyright (c) 2023, NVIDIA CORPORATION.
3+
4+
set -eoxu pipefail
5+
6+
# Download the pylibcugraph built in the previous step
7+
RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
8+
RAPIDS_PY_WHEEL_NAME="pylibcugraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-pylibcugraph-dep
9+
python -m pip install --no-deps ./local-pylibcugraph-dep/pylibcugraph*.whl
10+
11+
# Always install latest dask for testing
12+
python -m pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.10
13+
14+
# Only download test data for x86
15+
arch=$(uname -m)
16+
if [[ "${arch}" == "x86_64" ]]; then
17+
pushd ./datasets
18+
bash ./get_test_data.sh
19+
popd
20+
fi
21+
22+
./ci/test_wheel.sh cugraph python/cugraph

0 commit comments

Comments
 (0)