Allow cugraph to be imported in an SG env for SG algorithms (#2241)

This PR aims to directly resolve #2113, and allows for cugraph to be imported without errors in environments with only a single GPU

What this PR does:

- Moves `comms` module into the `dask` module, renaming imports across `cugraph` accordingly
- Adds mock class in case ucp is not available or usable in SG-only environments (note ucp requires libnuma.so)
- Renames `tests/dask` to `tests/mg` to better reflect the fact that those tests are for mg algs and utils
- Separates symmetrize testing into `tests/mg` and `tests` for separation of SG and MG testing

Authors:
  - https://github.com/betochimas

Approvers:
  - Rick Ratzel (https://github.com/rlratzel)
  - AJ Schmidt (https://github.com/ajschmidt8)

URL: #2241

Author: betochimas

ci/test.sh

@@ -91,7 +91,8 @@ echo "Ran Python pytest for pylibcugraph : return code was: $?, test script exit
 
 echo "Python pytest for cuGraph (single-GPU only)..."
 cd ${CUGRAPH_ROOT}/python/cugraph/cugraph
-pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph-pytests.xml -v --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term --ignore=raft --ignore=tests/dask --benchmark-disable
+# rmat is not tested because of MG testing
+pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph-pytests.xml -v --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term -k "not mg and not rmat" --benchmark-disable
 echo "Ran Python pytest for cugraph : return code was: $?, test script exit code is now: $EXITCODE"
 
 echo "Python benchmarks for cuGraph (running as tests)..."

docs/cugraph/source/api_docs/dask-cugraph.rst

@@ -105,7 +105,7 @@
 from cugraph.linear_assignment import hungarian, dense_hungarian
 from cugraph.layout import force_atlas2
 from raft import raft_include_test
-from cugraph.comms import comms
+from cugraph.dask.comms import comms
 
 from cugraph.sampling import random_walks, rw_path, node2vec
 

docs/cugraph/source/api_docs/helper_functions.rst

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -22,7 +22,7 @@ from libc.stdint cimport uintptr_t
 from libcpp cimport bool
 import cudf
 import numpy as np
-import cugraph.comms.comms as Comms
+import cugraph.dask.comms.comms as Comms
 from cugraph.dask.common.mg_utils import get_client
 import dask.distributed
 

python/cugraph/cugraph/__init__.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2020-2021, NVIDIA CORPORATION.
+# Copyright (c) 2020-2022, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -24,7 +24,7 @@ from libcpp cimport bool
 import cudf
 import numpy as np
 from cugraph.dask.common.mg_utils import get_client
-import cugraph.comms.comms as Comms
+import cugraph.dask.comms.comms as Comms
 import dask.distributed
 
 

python/cugraph/cugraph/centrality/betweenness_centrality_wrapper.pyx

@@ -18,7 +18,7 @@
                                              get_vertex_partition_offsets)
 from cugraph.dask.centrality import\
     mg_katz_centrality_wrapper as mg_katz_centrality
-import cugraph.comms.comms as Comms
+import cugraph.dask.comms.comms as Comms
 import dask_cudf
 
 

python/cugraph/cugraph/centrality/edge_betweenness_centrality_wrapper.pyx

@@ -19,8 +19,17 @@
 from dask_cudf.core import DataFrame as dcDataFrame
 from dask_cudf.core import Series as daskSeries
 
-import cugraph.comms.comms as Comms
-from raft.dask.common.utils import get_client
+import cugraph.dask.comms.comms as Comms
+# FIXME: this raft import breaks the library if ucx-py is
+# not available. They are necessary only when doing MG work.
+from cugraph.dask.common.read_utils import MissingUCXPy
+try:
+    from raft.dask.common.utils import get_client
+except ModuleNotFoundError as err:
+    if err.name == "ucp":
+        get_client = MissingUCXPy()
+    else:
+        raise
 from cugraph.dask.common.part_utils import _extract_partitions
 from dask.distributed import default_client
 from toolz import first

python/cugraph/cugraph/dask/centrality/katz_centrality.py

@@ -18,12 +18,21 @@
 from dask_cuda import LocalCUDACluster
 from dask.distributed import Client
 
-from raft.dask.common.utils import default_client
+# FIXME: this raft import breaks the library if ucx-py is
+# not available. They are necessary only when doing MG work.
+from cugraph.dask.common.read_utils import MissingUCXPy
+try:
+    from raft.dask.common.utils import default_client
+except ModuleNotFoundError as err:
+    if err.name == "ucp":
+        default_client = MissingUCXPy()
+    else:
+        raise
 # FIXME: cugraph/__init__.py also imports the comms module, but
 # depending on the import environment, cugraph/comms/__init__.py
 # may be imported instead. The following imports the comms.py
 # module directly
-from cugraph.comms import comms as Comms
+from cugraph.dask.comms import comms as Comms
 
 
 # FIXME: We currently look for the default client from dask, as such is the

python/cugraph/cugraph/dask/common/input_utils.py

@@ -21,7 +21,7 @@
 from dask_cudf.core import DataFrame as daskDataFrame
 from dask_cudf.core import Series as daskSeries
 from functools import reduce
-import cugraph.comms.comms as Comms
+import cugraph.dask.comms.comms as Comms
 from dask.delayed import delayed
 import cudf