Branch 23.06 resolve merge conflict for forward merge

.github/workflows/pr.yaml

@@ -121,9 +121,9 @@ jobs:
       build_type: pull-request
       package-name: cugraph
       # Always want to test against latest dask/distributed.
-      test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
+      test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
       # Skip dataset downloads on arm to save CI time -- arm only runs smoke tests.
       # On arm also need to install cupy from the specific site.
-      test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
+      test-before-arm64: "RAPIDS_PY_WHEEL_NAME=pylibcugraph_cu11 rapids-download-wheels-from-s3 ./local-pylibcugraph-dep && pip install --no-deps ./local-pylibcugraph-dep/*.whl && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
       test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v -m sg ./python/cugraph/cugraph/tests"
       test-smoketest: "python ci/wheel_smoke_test_cugraph.py"

.github/workflows/test.yaml

@@ -52,7 +52,7 @@ jobs:
       sha: ${{ inputs.sha }}
       package-name: cugraph
       # Always want to test against latest dask/distributed.
-      test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
+      test-before-amd64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
       # On arm also need to install cupy from the specific webpage.
-      test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@main git+https://github.com/dask/distributed.git@main git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
+      test-before-arm64: "cd ./datasets && bash ./get_test_data.sh && cd - && pip install 'cupy-cuda11x<12.0.0' -f https://pip.cupy.dev/aarch64 && pip install git+https://github.com/dask/dask.git@2023.3.2 git+https://github.com/dask/distributed.git@2023.3.2.1 git+https://github.com/rapidsai/dask-cuda.git@branch-23.06"
       test-unittest: "RAPIDS_DATASET_ROOT_DIR=/__w/cugraph/cugraph/datasets pytest -v -m sg ./python/cugraph/cugraph/tests"

.pre-commit-config.yaml

@@ -51,7 +51,7 @@ repos:
         pass_filenames: false
         additional_dependencies: [gitpython]
   - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.4.0
+    rev: v1.5.1
     hooks:
         - id: rapids-dependency-file-generator
           args: ["--clean"]

benchmarks/cugraph-dgl/notebooks/get_node_storage.ipynb

@@ -32,7 +32,8 @@
     "import cupy as cp\n",
     "\n",
     "#TODO: Enable in torch nightly\n",
-    "# torch.cuda.memory.change_current_allocator(rmm.rmm_torch_allocator)\n",
+    "# from rmm.allocators.torch import rmm_torch_allocator\n",
+    "# torch.cuda.memory.change_current_allocator(rmm_torch_allocator)\n",
     "        \n",
     "import cugraph_dgl\n",
     "from dgl.data import AsNodePredDataset\n",
@@ -92,8 +93,9 @@
     "    client = Client(cluster)\n",
     "    Comms.initialize(p2p=True)\n",
     "else:\n",
+    "    from rmm.allocators.torch import rmm_torch_allocator\n",
     "    rmm.reinitialize(pool_allocator=True, initial_pool_size=5e9, maximum_pool_size=20e9)\n",
-    "    torch.cuda.memory.change_current_allocator(rmm.rmm_torch_allocator)"
+    "    torch.cuda.memory.change_current_allocator(rmm_torch_allocator)"
    ]
   },
   {

benchmarks/cugraph-dgl/notebooks/heterogeneous_dataloader_benchmark.ipynb

@@ -32,7 +32,8 @@
     "import cupy as cp\n",
     "\n",
     "#TODO: Enable in torch nightly\n",
-    "# torch.cuda.memory.change_current_allocator(rmm.rmm_torch_allocator)\n",
+    "# from rmm.allocators.torch import rmm_torch_allocator\n",
+    "# torch.cuda.memory.change_current_allocator(rmm_torch_allocator)\n",
     "        \n",
     "import cugraph_dgl\n",
     "from dgl.data import AsNodePredDataset\n",
@@ -90,7 +91,8 @@
     "else:\n",
     "    enable_cudf_spilling()\n",
     "    rmm.reinitialize(pool_allocator=True, initial_pool_size=5e9, maximum_pool_size=20e9)\n",
-    "    #torch.cuda.memory.change_current_allocator(rmm.rmm_torch_allocator)"
+    "    # from rmm.allocators.torch import rmm_torch_allocator\n",
+    "    # torch.cuda.memory.change_current_allocator(rmm_torch_allocator)"
    ]
   },
   {

benchmarks/cugraph-dgl/notebooks/homogenous_dataloader_benchmark.ipynb

@@ -39,7 +39,8 @@
     "import numpy as np\n",
     "\n",
     "#TODO: Enable in torch nightly\n",
-    "# torch.cuda.memory.change_current_allocator(rmm.rmm_torch_allocator)\n",
+    "# from rmm.allocators.torch import rmm_torch_allocator\n",
+    "# torch.cuda.memory.change_current_allocator(rmm_torch_allocator)\n",
     "        \n",
     "import cugraph_dgl\n",
     "from dgl.data import AsNodePredDataset\n",
@@ -97,7 +98,8 @@
     "else:\n",
     "    enable_cudf_spilling()\n",
     "    rmm.reinitialize(pool_allocator=True, initial_pool_size=5e9, maximum_pool_size=20e9)\n",
-    "    #torch.cuda.memory.change_current_allocator(rmm.rmm_torch_allocator)"
+    "    # from rmm.allocators.torch import rmm_torch_allocator\n",
+    "    # torch.cuda.memory.change_current_allocator(rmm_torch_allocator)"
    ]
   },
   {

benchmarks/cugraph/pytest-based/bench_algos.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2022, NVIDIA CORPORATION.
+# Copyright (c) 2020-2023, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -268,7 +268,7 @@ def bench_graph_degrees(gpubenchmark, anyGraphWithAdjListComputed):
 
 def bench_betweenness_centrality(gpubenchmark, anyGraphWithAdjListComputed):
     gpubenchmark(cugraph.betweenness_centrality,
-                 anyGraphWithAdjListComputed, k=10, seed=123)
+                 anyGraphWithAdjListComputed, k=10, random_state=123)
 
 
 def bench_edge_betweenness_centrality(gpubenchmark,

benchmarks/cugraph/pytest-based/bench_cugraph_uniform_neighbor_sample.py

@@ -65,7 +65,7 @@ def create_graph(graph_data):
         # FIXME: edgelist_df should have column names that match the defaults
         # for G.from_cudf_edgelist()
         G.from_cudf_edgelist(
-            edgelist_df, source="src", destination="dst", edge_attr="wgt", legacy_renum_only=True
+            edgelist_df, source="src", destination="dst", edge_attr="wgt"
         )
         num_verts = G.number_of_vertices()
 
@@ -94,7 +94,6 @@ def create_graph(graph_data):
             source="src",
             destination="dst",
             edge_attr="weight",
-            legacy_renum_only=True,
         )
 
     else:
@@ -135,7 +134,7 @@ def create_mg_graph(graph_data):
         # for G.from_cudf_edgelist()
         edgelist_df = dask_cudf.from_cudf(edgelist_df)
         G.from_dask_cudf_edgelist(
-            edgelist_df, source="src", destination="dst", edge_attr="wgt", legacy_renum_only=True
+            edgelist_df, source="src", destination="dst", edge_attr="wgt"
         )
         num_verts = G.number_of_vertices()
 
@@ -164,7 +163,6 @@ def create_mg_graph(graph_data):
             source="src",
             destination="dst",
             edge_attr="weight",
-            legacy_renum_only=True,
         )
 
     else:

benchmarks/cugraph/standalone/benchmark.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.
+# Copyright (c) 2021-2023, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -133,7 +133,6 @@ def run(self):
         # Algos with transposed=True : PageRank, Katz.
         # Algos with transposed=False: BFS, SSSP, Louvain, HITS,
         # Neighborhood_sampling.
-        # Algos supporting the legacy_renum_only: HITS, Neighborhood_sampling
         #
         for i in range(len(self.algos)):
             # set transpose=True when renumbering
@@ -151,7 +150,7 @@ def run(self):
                         self.algos[i][1]["alpha"] = katz_alpha
                     if hasattr(G, "compute_renumber_edge_list"):
                         G.compute_renumber_edge_list(
-                            transposed=True, legacy_renum_only=True)
+                            transposed=True)
                 else:
                     # FIXME: Pagerank still follows the old path. Update this once it
                     # follows the pylibcugraph/C path
@@ -166,7 +165,7 @@ def run(self):
                         G.compute_renumber_edge_list(transposed=False)
                     else:
                         G.compute_renumber_edge_list(
-                            transposed=False, legacy_renum_only=True)
+                            transposed=False)
                 self.__log("done.")
         # FIXME: need to handle individual algo args
         for ((algo, params), validator) in zip(self.algos, self.validators):

benchmarks/shared/build_cugraph_ucx/test_cugraph_sampling.py

@@ -64,7 +64,6 @@ def create_mg_graph(graph_data):
         source="src",
         destination="dst",
         edge_attr="weight",
-        legacy_renum_only=True,
     )
     return G
 

ci/build_docs.sh

@@ -45,6 +45,9 @@ popd
 
 rapids-logger "Build Sphinx docs"
 pushd docs/cugraph
+# Ensure cugraph is importable, since sphinx does not report details about this
+# type of failure well.
+python -c "import cugraph; print(f'Using cugraph: {cugraph}')"
 sphinx-build -b dirhtml source _html
 sphinx-build -b text source _text
 popd

ci/release/apply_wheel_modifications.sh

@@ -28,3 +28,4 @@ sed -i "s/cudf/cudf${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml
 sed -i "s/raft-dask/raft-dask${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml
 sed -i "s/pylibcugraph/pylibcugraph${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml
 sed -i "s/pylibraft/pylibraft${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml
+sed -i "s/ucx-py/ucx-py${CUDA_SUFFIX}/g" python/cugraph/pyproject.toml

ci/test_python.sh

@@ -117,10 +117,11 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
       --channel "${PYTHON_CHANNEL}" \
       --channel pytorch \
       --channel pytorch-nightly \
-      --channel dglteam/label/cu117 \
+      --channel dglteam/label/cu118 \
       --channel nvidia \
       libcugraph \
       pylibcugraph \
+      pylibcugraphops \
       cugraph \
       cugraph-dgl \
       'dgl>=1.0' \
@@ -134,7 +135,6 @@ if [[ "${RAPIDS_CUDA_VERSION}" == "11.8.0" ]]; then
     pytest \
       --cache-clear \
       --ignore=mg \
-      --ignore=nn \
       --junitxml="${RAPIDS_TESTS_DIR}/junit-cugraph-dgl.xml" \
       --cov-config=../../.coveragerc \
       --cov=cugraph_dgl \

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -10,47 +10,51 @@ dependencies:
 - aiohttp
 - c-compiler
 - cmake>=3.23.1,!=3.25.0
-- cuda-python>=11.7.1,<12.0
 - cudatoolkit=11.8
-- cudf=23.06.*
+- cudf==23.6.*
+- cupy>=9.5.0,<12.0.0a0
 - cxx-compiler
 - cython>=0.29,<0.30
-- dask-cuda=23.06.*
-- dask-cudf=23.06.*
-- dask==2023.1.1
-- distributed==2023.1.1
+- dask-core==2023.3.2
+- dask-cuda==23.6.*
+- dask-cudf==23.6.*
+- dask==2023.3.2
+- distributed==2023.3.2.1
 - doxygen
+- fsspec[http]>=0.6.0
 - gcc_linux-64=11.*
 - gmock=1.10.0
 - graphviz
 - gtest=1.10.0
 - ipython
-- libcudf=23.06.*
-- libcugraphops=23.06.*
-- libraft-headers=23.06.*
-- libraft=23.06.*
-- librmm=23.06.*
+- libcudf=23.6.*
+- libcugraphops=23.6.*
+- libraft-headers=23.6.*
+- libraft=23.6.*
+- librmm=23.6.*
 - nbsphinx
 - nccl>=2.9.9
 - networkx>=2.5.1
 - ninja
 - notebook>=0.5.0
+- numba>=0.56.2
+- numpy>=1.21
 - numpydoc
 - nvcc_linux-64=11.8
 - openmpi
-- pip
+- pandas
 - pre-commit
-- py
 - pydata-sphinx-theme
-- pylibraft=23.06.*
+- pylibraft==23.6.*
 - pytest
+- pytest-benchmark
 - pytest-cov
+- pytest-xdist
 - python-louvain
-- raft-dask=23.06.*
-- rapids-pytest-benchmark
+- raft-dask==23.6.*
 - recommonmark
 - requests
-- rmm=23.06.*
+- rmm==23.6.*
 - scikit-build>=0.13.1
 - scikit-learn>=0.23.1
 - scipy
@@ -59,5 +63,5 @@ dependencies:
 - sphinx<6
 - sphinxcontrib-websupport
 - ucx-proc=*=gpu
-- ucx-py=0.32.*
+- ucx-py==0.32.*
 name: all_cuda-118_arch-x86_64

conda/recipes/cugraph-pyg/meta.yaml

@@ -26,13 +26,13 @@ requirements:
     - python x.x
     - scikit-build >=0.13.1
   run:
-    - distributed >=2023.1.1
+    - distributed ==2023.3.2.1
     - numba >=0.56.2
     - numpy
     - pytorch >=2.0
     - cupy >=9.5.0,<12.0.0a0
     - cugraph ={{ version }}
-    - pyg =2.3
+    - pyg >=2.3,<2.4
 
 tests:
   imports:

conda/recipes/cugraph-service/meta.yaml

@@ -57,7 +57,7 @@ outputs:
         - cupy >=9.5.0,<12.0.0a0
         - dask-cuda ={{ minor_version }}
         - dask-cudf ={{ minor_version }}
-        - distributed >=2023.1.1
+        - distributed ==2023.3.2.1
         - numpy
         - python x.x
         - thriftpy2 >=0.4.15

conda/recipes/cugraph/meta.yaml

@@ -72,8 +72,9 @@ requirements:
     - cupy >=9.5.0,<12.0.0a0
     - dask-cuda ={{ minor_version }}
     - dask-cudf ={{ minor_version }}
-    - dask >=2023.1.1
-    - distributed >=2023.1.1
+    - dask ==2023.3.2
+    - dask-core ==2023.3.2
+    - distributed ==2023.3.2.1
     - libcugraph ={{ version }}
     - libraft ={{ minor_version }}
     - libraft-headers ={{ minor_version }}

cpp/CMakeLists.txt

@@ -186,6 +186,10 @@ set(CUGRAPH_SOURCES
     src/sampling/random_walks_mg.cu
     src/community/detail/common_methods_mg.cu
     src/community/detail/common_methods_sg.cu
+    src/community/detail/refine_sg.cu
+    src/community/detail/refine_mg.cu
+    src/community/detail/mis_sg.cu
+    src/community/detail/mis_mg.cu
     src/detail/utility_wrappers.cu
     src/structure/graph_view_mg.cu
     src/utilities/cython.cu
@@ -207,6 +211,7 @@ set(CUGRAPH_SOURCES
     src/community/louvain_sg.cu
     src/community/louvain_mg.cu
     src/community/leiden_sg.cu
+    src/community/leiden_mg.cu
     src/community/legacy/louvain.cu
     src/community/legacy/leiden.cu
     src/community/legacy/ktruss.cu