Merge remote-tracking branch 'origin/branch-0.8' into fea-ext-memory-resource

Author: jrhemstad

.gitignore

@@ -41,3 +41,6 @@ include/rmm/detail/cnmem.h
 
 # generated librmm_build.py
 python/librmm_cffi/librmm_build.py
+
+# Doxygen 
+doxygen/html/

CHANGELOG.md

@@ -2,20 +2,62 @@
 ## New Features
 
  - PR #96 Added `device_memory_resource` for beginning of overhaul of RMM design
- 
+
 ## Improvements
 
+    ...
+
 ## Bug Fixes
 
+ - PR #92 Update docs version
 
-# RMM 0.5.0 (Date TBD)
+
+# RMM 0.7.0 (10 May 2019)
+
+## New Features
+
+ - PR #67 Add random_allocate microbenchmark in tests/performance
+ - PR #70 Create conda environments and conda recipes
+ - PR #77 Add local build script to mimic gpuCI
+ - PR #80 Add build script for docs
+
+## Improvements
+
+- PR #76 Add cudatoolkit conda dependency
+- PR #84 Use latest release version in update-version CI script
+- PR #90 Avoid using c++14 auto return type for thrust_rmm_allocator.h 
+
+## Bug Fixes
+
+ - PR #68 Fix signed/unsigned mismatch in random_allocate benchmark
+ - PR #74 Fix rmm conda recipe librmm version pinning
+ - PR #72 Remove unnecessary _BSD_SOURCE define in random_allocate.cpp
+
+
+# RMM 0.6.0 (18 Mar 2019)
+
+## New Features
+
+ - PR #43 Add gpuCI build & test scripts
+ - PR #44 Added API to query whether RMM is initialized and with what options.
+ - PR #60 Default to CXX11_ABI=ON
+
+## Improvements
+
+## Bug Fixes
+
+ - PR #58 Eliminate unreliable check for change in available memory in test
+ - PR #49 Fix pep8 style errors detected by flake8
+
+
+# RMM 0.5.0 (28 Jan 2019)
 
 ## New Features
 
  - PR #2 Added CUDA Managed Memory allocation mode
- 
+
 ## Improvements
- 
+
  - PR #12 Enable building RMM as a submodule
  - PR #13 CMake: Added CXX11ABI option and removed Travis references
  - PR #16 CMake: Added PARALLEL_LEVEL environment variable handling for GTest build parallelism (matches cuDF)
@@ -25,6 +67,8 @@
 
  - PR #10 Change cnmem submodule URL to use https
  - PR #15 Temporarily disable hanging AllocateTB test for managed memory
+ - PR #28 Fix invalid reference to local stack variable in `rmm::exec_policy`
+
 
 # RMM 0.4.0 (20 Dec 2018)
 
@@ -36,11 +80,11 @@
 
 - CUDF PR #472 RMM: Created centralized rmm::device_vector alias and rmm::exec_policy
 - CUDF PR #465 Added templated C++ API for RMM to avoid explicit cast to `void**`
- 
+
 ## Bug Fixes
 
 
-RMM was initially implemented as part of cuDF, so we include the relevant changelog history below. 
+RMM was initially implemented as part of cuDF, so we include the relevant changelog history below.
 
 # cuDF 0.3.0 (23 Nov 2018)
 
@@ -49,7 +93,7 @@ RMM was initially implemented as part of cuDF, so we include the relevant change
  - PR #336 CSV Reader string support
 
 ## Improvements
- 
+
  - CUDF PR #333 Add Rapids Memory Manager documentation
  - CUDF PR #321 Rapids Memory Manager adds file/line location logging and convenience macros
 

CMakeLists.txt

@@ -15,7 +15,7 @@
 #=============================================================================
 cmake_minimum_required(VERSION 3.12 FATAL_ERROR)
 
-project(RMM VERSION 0.4.0 LANGUAGES C CXX CUDA)
+project(RMM VERSION 0.7.0 LANGUAGES C CXX CUDA)
 
 ###################################################################################################
 # - build type ------------------------------------------------------------------------------------
@@ -43,13 +43,14 @@ set(CMAKE_CXX_COMPILER $ENV{CXX})
 if(CMAKE_COMPILER_IS_GNUCXX)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Werror")
 
-    option(CMAKE_CXX11_ABI "Enable the GLIBCXX11 ABI" OFF)
+    option(CMAKE_CXX11_ABI "Enable the GLIBCXX11 ABI" ON)
     if(CMAKE_CXX11_ABI)
-        message(STATUS "CUDF: Enabling the GLIBCXX11 ABI")
+        message(STATUS "RMM: Enabling the GLIBCXX11 ABI")
     else()
-        message(STATUS "CUDF: Disabling the GLIBCXX11 ABI")
+        message(STATUS "RMM: Disabling the GLIBCXX11 ABI")
         set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
+        set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -Xcompiler -D_GLIBCXX_USE_CXX11_ABI=0")
     endif(CMAKE_CXX11_ABI)
 endif(CMAKE_COMPILER_IS_GNUCXX)
 
@@ -168,3 +169,13 @@ add_custom_command(OUTPUT RMM_INSTALL_PYTHON_CFFI
                    VERBATIM)
 
 add_custom_target(rmm_install_python DEPENDS rmm RMM_PYTHON_CFFI RMM_INSTALL_PYTHON_CFFI)
+
+###################################################################################################
+# - make documentation ----------------------------------------------------------------------------
+
+add_custom_command(OUTPUT RMM_DOXYGEN
+                   WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/doxygen
+                   COMMAND doxygen Doxyfile
+                   VERBATIM)
+
+add_custom_target(rmm_doc DEPENDS RMM_DOXYGEN)

CONTRIBUTING.md

@@ -49,5 +49,10 @@ contributing to. Start with _Step 3_ from above, commenting on the issue to let
 others know you are working on it. If you have any questions related to the
 implementation of the issue, ask them in the issue instead of the PR.
 
+### Building and Testing on a gpuCI image locally
+
+ Before submitting a pull request, you can do a local build and test on your machine that mimics our gpuCI environment using the `ci/local/build.sh` script.
+For detailed information on usage of this script, see [here](ci/local/README.md).
+
 ## Attribution
 Portions adopted from https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md

README.md

@@ -121,7 +121,6 @@ situations:
 
 RMM includes a custom Thrust allocator in the file `thrust_rmm_allocator.h`. This defines the template class `rmm_allocator`, and 
 a custom Thrust CUDA device execution policy called `rmm::exec_policy(stream)`.
-```
 
 #### Thrust Device Vectors
 

ci/checks/changelog.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+#########################
+# RMM CHANGELOG Tester #
+#########################
+
+# Checkout master for comparison
+git checkout --quiet master
+
+# Switch back to tip of PR branch
+git checkout --quiet current-pr-branch
+
+# Ignore errors during searching
+set +e
+
+# Get list of modified files between matster and PR branch
+CHANGELOG=`git diff --name-only master...current-pr-branch | grep CHANGELOG.md`
+# Check if CHANGELOG has PR ID
+PRNUM=`cat CHANGELOG.md | grep "$PR_ID"`
+RETVAL=0
+
+# Return status of check result
+if [ "$CHANGELOG" != "" -a "$PRNUM" != "" ] ; then
+  echo -e "\n\n>>>> PASSED: CHANGELOG.md has been updated with current PR information.\n\nPlease ensure the update meets the following criteria.\n"
+else
+  echo -e "\n\n>>>> FAILED: CHANGELOG.md has not been updated!\n\nPlease add a line describing this PR to CHANGELOG.md in the repository root directory. The line should meet the following criteria.\n"
+  RETVAL=1
+fi
+
+cat << EOF
+  It should be placed under the section for the appropriate release.
+  It should be placed under "New Features", "Improvements", or "Bug Fixes" as appropriate.
+  It should be formatted as '- PR #<PR number> <Concise human-readable description of the PR's new feature, improvement, or bug fix>'
+    Example format for #491 '- PR #491 Add CI test script to check for updates to CHANGELOG.md in PRs'
+
+
+EOF
+
+exit $RETVAL

ci/checks/style.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+#####################
+# RMM Style Tester #
+#####################
+
+# Ignore errors and set path
+set +e
+PATH=/conda/bin:$PATH
+
+# Activate common conda env
+source activate gdf
+
+# Run flake8 and get results/return code
+FLAKE=`flake8 python`
+RETVAL=$?
+
+# Output results if failure otherwise show pass
+if [ "$FLAKE" != "" ]; then
+  echo -e "\n\n>>>> FAILED: flake8 style check; begin output\n\n"
+  echo -e "$FLAKE"
+  echo -e "\n\n>>>> FAILED: flake8 style check; end output\n\n"
+else
+  echo -e "\n\n>>>> PASSED: flake8 style check\n\n"
+fi
+
+exit $RETVAL

ci/cpu/build.sh

@@ -0,0 +1,79 @@
+#!/usr/bin/env bash
+# Copyright (c) 2018, NVIDIA CORPORATION.
+######################################
+# rmm CPU build script for CI        #
+######################################
+set -e
+
+# Logger function for build status output
+function logger() {
+  echo -e "\n>>>> $@\n"
+}
+
+# Set path and build parallel level
+export PATH=/conda/bin:/usr/local/cuda/bin:$PATH
+export PARALLEL_LEVEL=4
+
+# Set home to the job's workspace
+export HOME=$WORKSPACE
+
+# Switch to project root; also root of repo checkout
+cd $WORKSPACE
+
+# Get latest tag and number of commits since tag
+export GIT_DESCRIBE_TAG=`git describe --abbrev=0 --tags`
+export GIT_DESCRIBE_NUMBER=`git rev-list ${GIT_DESCRIBE_TAG}..HEAD --count`
+
+################################################################################
+# SETUP - Check environment
+################################################################################
+
+logger "Get env..."
+env
+
+logger "Activate conda env..."
+source activate gdf
+
+logger "Check versions..."
+python --version
+gcc --version
+g++ --version
+conda list
+
+# FIX Added to deal with Anancoda SSL verification issues during conda builds
+conda config --set ssl_verify False
+
+################################################################################
+# INSTALL - Install NVIDIA driver
+################################################################################
+
+logger "Install NVIDIA driver for CUDA $CUDA..."
+apt-get update -q
+DRIVER_VER="396.44-1"
+LIBCUDA_VER="396"
+if [ "$CUDA" == "10.0" ]; then
+  DRIVER_VER="410.72-1"
+  LIBCUDA_VER="410"
+fi
+DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+  cuda-drivers=${DRIVER_VER} libcuda1-${LIBCUDA_VER}
+
+################################################################################
+# BUILD - Conda package builds (conda deps: librmm <- rmm)
+################################################################################
+
+logger "Build conda pkg for librmm..."
+source ci/cpu/librmm/build_librmm.sh
+
+logger "Build conda pkg for rmm..."
+source ci/cpu/rmm/build_rmm.sh
+
+################################################################################
+# UPLOAD - Conda packages
+################################################################################
+
+logger "Upload conda pkg for librmm..."
+source ci/cpu/librmm/upload-anaconda.sh
+
+logger "Upload conda pkg for rmm..."
+source ci/cpu/rmm/upload-anaconda.sh

ci/cpu/librmm/build_librmm.sh

@@ -0,0 +1,12 @@
+set -e
+
+if [ "$BUILD_LIBRMM" == '1' ]; then
+  echo "Building librmm"
+  CUDA_REL=${CUDA:0:3}
+  if [ "${CUDA:0:2}" == '10' ]; then
+    # CUDA 10 release
+    CUDA_REL=${CUDA:0:4}
+  fi
+
+  conda build conda/recipes/librmm -c nvidia/label/cuda${CUDA_REL} -c rapidsai/label/cuda${CUDA_REL} -c rapidsai-nightly/label/cuda${CUDA_REL} -c conda-forge --python=$PYTHON
+fi

ci/cpu/librmm/upload-anaconda.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Adopted from https://github.com/tmcdonell/travis-scripts/blob/dfaac280ac2082cd6bcaba3217428347899f2975/update-accelerate-buildbot.sh
+
+set -e
+
+if [ "$BUILD_LIBRMM" == '1' ]; then
+  export UPLOADFILE=`conda build conda/recipes/librmm -c rapidsai -c rapidsai-nightly -c nvidia -c conda-forge --python=$PYTHON --output`
+
+  SOURCE_BRANCH=master
+  CUDA_REL=${CUDA:0:3}
+  if [ "${CUDA:0:2}" == '10' ]; then
+    # CUDA 10 release
+    CUDA_REL=${CUDA:0:4}
+  fi
+
+  SOURCE_BRANCH=master
+
+  LABEL_OPTION="--label main --label cuda${CUDA_REL}"
+  echo "LABEL_OPTION=${LABEL_OPTION}"
+
+  # Restrict uploads to master branch
+  if [ ${GIT_BRANCH} != ${SOURCE_BRANCH} ]; then
+    echo "Skipping upload"
+    return 0
+  fi
+
+  if [ -z "$MY_UPLOAD_KEY" ]; then
+      echo "No upload key"
+      return 0
+  fi
+
+  echo "Upload"
+  echo ${UPLOADFILE}
+  anaconda -t ${MY_UPLOAD_KEY} upload -u ${CONDA_USERNAME:-rapidsai} ${LABEL_OPTION} --force ${UPLOADFILE}
+fi