Adds setup.py files and conda recipes for cugraph-service

build.sh

@@ -20,16 +20,17 @@ REPODIR=$(cd $(dirname $0); pwd)
 LIBCUGRAPH_BUILD_DIR=${LIBCUGRAPH_BUILD_DIR:=${REPODIR}/cpp/build}
 LIBCUGRAPH_ETL_BUILD_DIR=${LIBCUGRAPH_ETL_BUILD_DIR:=${REPODIR}/cpp/libcugraph_etl/build}
 
-VALIDARGS="clean uninstall uninstall_cmake_deps libcugraph libcugraph_etl cugraph pylibcugraph cpp-mgtests docs -v -g -n --pydevelop --allgpuarch --skip_cpp_tests --cmake_default_generator -h --help"
+VALIDARGS="clean uninstall uninstall_cmake_deps libcugraph libcugraph_etl cugraph cugraph-service pylibcugraph cpp-mgtests docs -v -g -n --pydevelop --allgpuarch --skip_cpp_tests --cmake_default_generator -h --help"
 HELP="$0 [<target> ...] [<flag> ...]
  where <target> is:
    clean                      - remove all existing build artifacts and configuration (start over)
    uninstall                  - uninstall libcugraph and cugraph from a prior build/install (see also -n)
    uninstall_cmake_deps       - uninstall headers from external dependencies installed by cmake (raft, rmm, cuco, etc.) (see also -n)
    libcugraph                 - build libcugraph.so and SG test binaries
    libcugraph_etl             - build libcugraph_etl.so and SG test binaries
-   cugraph                    - build the cugraph Python package
    pylibcugraph               - build the pylibcugraph Python package
+   cugraph                    - build the cugraph Python package
+   cugraph-service            - build the cugraph-service_client and cugraph-service_server Python package
    cpp-mgtests                - build libcugraph and libcugraph_etl MG tests. Builds MPI communicator, adding MPI as a dependency.
    docs                       - build the docs
  and <flag> is:
@@ -42,7 +43,7 @@ HELP="$0 [<target> ...] [<flag> ...]
    --cmake_default_generator  - use the default cmake generator instead of ninja
    -h                         - print this text
 
- default action (no args) is to build and install 'libcugraph' then 'libcugraph_etl' then 'pylibcugraph' then 'cugraph' targets
+ default action (no args) is to build and install 'libcugraph' then 'libcugraph_etl' then 'pylibcugraph' then 'cugraph' then 'cugraph-service' targets
 
  libcugraph build dir is: ${LIBCUGRAPH_BUILD_DIR}
 
@@ -74,7 +75,7 @@ function hasArg {
 }
 
 function buildAll {
-    (( ${NUMARGS} == 0 )) || !(echo " ${ARGS} " | grep -q " [^-]\+ ")
+    (( ${NUMARGS} == 0 )) || !(echo " ${ARGS} " | grep -q " [^-][a-zA-Z0-9\_\-]\+ ")
 }
 
 if hasArg -h || hasArg --help; then
@@ -265,6 +266,16 @@ if buildAll || hasArg cugraph; then
     fi
 fi
 
+# Install the cugraph-service-client and cugraph-service-server Python packages
+if buildAll || hasArg cugraph-service; then
+    if [[ ${INSTALL_TARGET} != "" ]]; then
+        cd ${REPODIR}/python/cugraph-service/client
+	python setup.py ${PYTHON_INSTALL}
+        cd ${REPODIR}/python/cugraph-service/server
+	python setup.py ${PYTHON_INSTALL}
+    fi
+fi
+
 # Build the docs
 if hasArg docs; then
     if [ ! -d ${LIBCUGRAPH_BUILD_DIR} ]; then

ci/checks/style.sh

@@ -62,7 +62,7 @@ fi
 
 # Check for copyright headers in the files modified currently
 #COPYRIGHT=`env PYTHONPATH=ci/utils python ci/checks/copyright.py cpp python benchmarks ci 2>&1`
-COPYRIGHT=`env PYTHONPATH=ci/utils python ci/checks/copyright.py --git-modified-only 2>&1`
+COPYRIGHT=`env PYTHONPATH=ci/utils python ci/checks/copyright.py --git-modified-only --exclude=".*/versioneer.py" 2>&1`
 CR_RETVAL=$?
 ERRORCODE=$((ERRORCODE | ${CR_RETVAL}))
 

ci/cpu/build.sh

@@ -107,13 +107,21 @@ else
 fi
 
 if [ "$BUILD_CUGRAPH" == "1" ]; then
-  gpuci_logger "Building conda packages for pylibcugraph and cugraph"
+  gpuci_logger "Building conda packages for pylibcugraph, cugraph, and cugraph-service"
   if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
+    gpuci_logger "pylibcugraph"
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pylibcugraph --python=$PYTHON
+    gpuci_logger "cugraph"
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cugraph --python=$PYTHON
+    gpuci_logger "cugraph-service"
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cugraph-service --python=$PYTHON
   else
+    gpuci_logger "pylibcugraph"
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/pylibcugraph -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON
+    gpuci_logger "cugraph"
     gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cugraph -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON
+    gpuci_logger "cugraph-service"
+    gpuci_conda_retry mambabuild --no-build-id --croot ${CONDA_BLD_DIR} conda/recipes/cugraph-service -c ${CONDA_LOCAL_CHANNEL} --dirty --no-remove-work-dir --python=$PYTHON
     mkdir -p ${CONDA_BLD_DIR}/cugraph
     mv ${CONDA_BLD_DIR}/work ${CONDA_BLD_DIR}/cugraph/work
   fi

ci/cpu/upload.sh

@@ -45,4 +45,7 @@ if [[ "$BUILD_CUGRAPH" == "1" ]]; then
   test -e ${CUGRAPH_FILE}
   echo "Upload cugraph file: ${CUGRAPH_FILE}"
   gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing ${CUGRAPH_FILE} --no-progress
+  CUGRAPH_SERVICE_FILES=$(conda build --croot ${CONDA_BLD_DIR} conda/recipes/cugraph-service --python=$PYTHON --output)
+  echo "Upload cugraph-server files: ${CUGRAPH_SERVICE_FILES}"
+  gpuci_retry anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --skip-existing --no-progress ${CUGRAPH_SERVICE_FILES}
 fi

ci/gpu/build.sh

@@ -114,7 +114,7 @@ conda list --show-channel-urls
 
 if [[ -z "$PROJECT_FLASH" || "$PROJECT_FLASH" == "0" ]]; then
     gpuci_logger "Build from source"
-    $WORKSPACE/build.sh -v clean libcugraph pylibcugraph cugraph
+    $WORKSPACE/build.sh -v clean libcugraph pylibcugraph cugraph cugraph-service-server cugraph-service-client
 else
     gpuci_logger "Installing libcugraph-tests"
     gpuci_mamba_retry install -c ${CONDA_ARTIFACT_PATH} libcugraph libcugraph_etl libcugraph-tests
@@ -125,11 +125,15 @@ else
     gpuci_logger "Building and installing pylibcugraph and cugraph..."
     export CONDA_BLD_DIR="${WORKSPACE}/.conda-bld"
     export VERSION_SUFFIX=""
+    gpuci_logger "pylibcugraph"
     gpuci_conda_retry mambabuild conda/recipes/pylibcugraph --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
+    gpuci_logger "cugraph"
     gpuci_conda_retry mambabuild conda/recipes/cugraph --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
+    gpuci_logger "cugraph-service"
+    gpuci_conda_retry mambabuild conda/recipes/cugraph-service --no-build-id --croot ${CONDA_BLD_DIR} -c ${CONDA_ARTIFACT_PATH} --python=${PYTHON}
 
-    gpuci_logger "Installing pylibcugraph and cugraph from build / artifact dirs"
-    gpuci_mamba_retry install -c ${CONDA_BLD_DIR} -c ${CONDA_ARTIFACT_PATH} --strict-channel-priority pylibcugraph cugraph
+    gpuci_logger "Installing pylibcugraph, cugraph, and cugraph-service from build / artifact dirs"
+    gpuci_mamba_retry install -c ${CONDA_BLD_DIR} -c ${CONDA_ARTIFACT_PATH} --strict-channel-priority pylibcugraph cugraph cugraph-service-server cugraph-service-client
 fi
 
 ################################################################################

ci/test.sh

@@ -87,20 +87,25 @@ fi
 if hasArg "--run-python-tests"; then
     echo "Python pytest for pylibcugraph..."
     cd ${CUGRAPH_ROOT}/python/pylibcugraph/pylibcugraph
-    pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-pylibcugraph-pytests.xml -v --cov-config=.coveragerc --cov=pylibcugraph --cov-report=xml:${WORKSPACE}/python/pylibcugraph/pylibcugraph-coverage.xml --cov-report term --ignore=raft --benchmark-disable
+    pytest -sv --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-pylibcugraph-pytests.xml --cov-config=.coveragerc --cov=pylibcugraph --cov-report=xml:${WORKSPACE}/python/pylibcugraph/pylibcugraph-coverage.xml --cov-report term --ignore=raft --benchmark-disable
     echo "Ran Python pytest for pylibcugraph : return code was: $?, test script exit code is now: $EXITCODE"
 
     echo "Python pytest for cuGraph (single-GPU only)..."
     conda list
     cd ${CUGRAPH_ROOT}/python/cugraph/cugraph
     # rmat is not tested because of MG testing
-    pytest --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph-pytests.xml -v --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term --ignore=raft --ignore=tests/mg --ignore=tests/generators --benchmark-disable
+    pytest -sv --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph-pytests.xml --cov-config=.coveragerc --cov=cugraph --cov-report=xml:${WORKSPACE}/python/cugraph/cugraph-coverage.xml --cov-report term --ignore=raft --ignore=tests/mg --ignore=tests/generators --benchmark-disable
     echo "Ran Python pytest for cugraph : return code was: $?, test script exit code is now: $EXITCODE"
 
     echo "Python benchmarks for cuGraph (running as tests)..."
     cd ${CUGRAPH_ROOT}/benchmarks
-    pytest -v -m "managedmem_on and poolallocator_on and tiny" --benchmark-disable
+    pytest -sv -m "managedmem_on and poolallocator_on and tiny" --benchmark-disable
     echo "Ran Python benchmarks for cuGraph (running as tests) : return code was: $?, test script exit code is now: $EXITCODE"
+
+    echo "Python pytest for cugraph-service (single-GPU only)..."
+    cd ${CUGRAPH_ROOT}/python/cugraph-service
+    pytest -sv --cache-clear --junitxml=${CUGRAPH_ROOT}/junit-cugraph-service-pytests.xml --benchmark-disable -k "not mg" ./tests
+    echo "Ran Python pytest for cugraph-service : return code was: $?, test script exit code is now: $EXITCODE"
 fi
 
 echo "Test script exiting with value: $EXITCODE"

conda/recipes/cugraph-service/install_cugraph-service-client.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+# This assumes the script is executed from the root of the repo directory
+
+# The standard "./build.sh cugraph_service" cannot be used here since a custom
+# setup.py invocation must be done.  This is required to work around the bug
+# with split packages described here:
+# https://github.com/conda/conda-build/issues/3993
+# This workaround was copied from this PR:
+# https://github.com/rapidsai/ucx-split-feedstock/pull/28
+
+cd "${SRC_DIR}/python/cugraph-service/client"
+mkdir -p pip_cache
+$PYTHON -m pip install --no-index --no-deps --ignore-installed --cache-dir ./pip_cache . -vv
+
+#./build.sh cugraph-service

conda/recipes/cugraph-service/install_cugraph-service-server.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2022, NVIDIA CORPORATION.
+
+# This assumes the script is executed from the root of the repo directory
+
+# The standard "./build.sh cugraph_service" cannot be used here since a custom
+# setup.py invocation must be done.  This is required to work around the bug
+# with split packages described here:
+# https://github.com/conda/conda-build/issues/3993
+# This workaround was copied from this PR:
+# https://github.com/rapidsai/ucx-split-feedstock/pull/28
+
+cd "${SRC_DIR}/python/cugraph-service/server"
+mkdir -p pip_cache
+$PYTHON -m pip install --no-index --no-deps --ignore-installed --cache-dir ./pip_cache . -vv
+
+#./build.sh cugraph-service

conda/recipes/cugraph-service/meta.yaml

@@ -0,0 +1,65 @@
+# Copyright (c) 2018-2022, NVIDIA CORPORATION.
+
+# Usage:
+#   conda build -c nvidia -c rapidsai -c conda-forge  .
+{% set version = environ.get('GIT_DESCRIBE_TAG', '0.0.0.dev').lstrip('v') + environ.get('VERSION_SUFFIX', '') %}
+{% set minor_version =  version.split('.')[0] + '.' + version.split('.')[1] %}
+
+package:
+  name: cugraph-service-split
+
+source:
+  git_url: ../../..
+
+outputs:
+  - name: cugraph-service-client
+    version: {{ version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_family: APACHE
+      license_file: ../../../LICENSE
+      summary: cugraph-service client
+    script: install_cugraph-service-client.sh
+    requirements:
+      host:
+        - python x.x
+        - pip
+        # FIXME: this pin can be removed once we move to the GitHub Actions build process
+        - setuptools<=65.2.0
+      run:
+        - python x.x
+        - thriftpy2 >=0.4.15
+
+  - name: cugraph-service-server
+    version: {{ version }}
+    about:
+      home: http://rapids.ai/
+      license: Apache-2.0
+      license_family: APACHE
+      license_file: ../../../LICENSE
+      summary: cugraph-service server
+    script: install_cugraph-service-server.sh
+    requirements:
+      host:
+        - python x.x
+        - pip
+        # FIXME: this pin can be removed once we move to the GitHub Actions build process
+        - setuptools<=65.2.0
+      run:
+        - python x.x
+        - thriftpy2 >=0.4.15
+        - {{ pin_compatible('cugraph-service-client', exact=True) }}
+        - {{ pin_compatible('cugraph', exact=True) }}
+        - cupy >=9.5.0,<12.0.0a0
+        - numpy
+        - ucx-py 0.29.*
+        - distributed >=2022.9.2
+        - dask-cuda {{ minor_version }}.*
+        - cudf {{ minor_version }}.*
+        - dask-cudf {{ minor_version }}.*
+
+tests:                                 # [linux64]
+  imports:                             # [linux64]
+    - cugraph_service_server           # [linux64]
+    - cugraph_service_client           # [linux64]

python/cugraph_service/README.md → python/cugraph-service/README.md

@@ -7,17 +7,14 @@
 
 -----
 
-## Server
-(description)
-### Installing `cugraph_service_server` conda package
-
-    TBD
-
-## Client
-(description)
-### Installing the `cugraph_service_client` conda package
-
-    TBD
+## Build & Install
+Build and install the client first, then the server. This is necessary because the server depends on shared modules provided by the client.
+```
+$> cd cugraph_repo/python/cugraph_service/client
+$> python setup.py install
+$> cd ../server
+$> python setup.py install
+```
 
 ### Example
 Starting a server for single-GPU-only cuGraph, using server extensions in `/my/cugraph_service/extensions`: