diff --git a/BUILD b/BUILD index 94c38a59b..f204015ad 100644 --- a/BUILD +++ b/BUILD @@ -8,6 +8,7 @@ test_suite( ":test_hive_hcatalog", ":test_hive_llap", ":test_starburst_presto", + ":test_spark_rapids", "//alluxio:test_alluxio", "//atlas:test_atlas", "//bigtable:test_bigtable", @@ -34,7 +35,6 @@ test_suite( "//solr:test_solr", "//sqoop:test_sqoop", "//tony:test_tony", - "//sparkRapids:test_sparkRapids", ], ) @@ -110,6 +110,22 @@ py_test( ], ) +py_test( + name = "test_spark_rapids", + size = "enormous", + srcs = ["spark-rapids/test_spark_rapids.py"], + data = [ + "spark-rapids/spark-rapids.sh", + "spark-rapids/verify_xgboost_spark_rapids.scala", + ], + local = True, + shard_count = 3, + deps = [ + "//integration_tests:dataproc_test_case", + "@io_abseil_py//absl/testing:parameterized", + ], +) + py_library( name = "pyspark_metastore_test", testonly = True, diff --git a/sparkRapids/README.md b/spark-rapids/README.md similarity index 76% rename from sparkRapids/README.md rename to spark-rapids/README.md index 875fa6f87..2f9fb1173 100644 --- a/sparkRapids/README.md +++ b/spark-rapids/README.md @@ -1,6 +1,6 @@ # SPARK-RAPIDS -The [RAPIDS Accelerator for Apache Spark](https://nvidia.github.io/spark-rapids/) leverages GPUs +The [RAPIDS Accelerator for Apache Spark](https://nvidia.github.io/spark-rapids/) leverages GPUs to accelerate processing via the [RAPIDS libraries](http://rapids.ai). This initialization action supports Spark runtimes for RAPIDS on [Google Cloud Dataproc](https://cloud.google.com/dataproc) cluster. @@ -28,7 +28,7 @@ To use RAPIDS Accelerator For Apache Spark, XGBoost4j with Spark 3 * NVIDIA GPU driver 440.33+ * CUDA v11.5/v11.0/v10.2/v10.1 * NCCL 2.11.4+ - * Ubuntu 18.04, Ubuntu 20.04 or Rocky Linux 7, Rocky Linux8, Debian 10 + * Ubuntu 18.04, Ubuntu 20.04 or Rocky Linux 7, Rocky Linux8, Debian 10, Debian 11 This section describes how to create [Google Cloud Dataproc](https://cloud.google.com/dataproc) cluster with @@ -67,7 +67,7 @@ gcloud dataproc clusters create $CLUSTER_NAME \ --worker-accelerator type=nvidia-tesla-t4,count=$NUM_GPUS \ --worker-machine-type n1-standard-8 \ --num-worker-local-ssds 1 \ - --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/sparkRapids/spark-rapids.sh \ + --initialization-actions gs://goog-dataproc-initialization-actions-${REGION}/spark-rapids/spark-rapids.sh \ --optional-components=JUPYTER,ZEPPELIN \ --metadata gpu-driver-provider="NVIDIA",rapids-runtime="SPARK",cuda-version="$CUDA_VER" \ --bucket $GCS_BUCKET \ @@ -114,4 +114,23 @@ In some releases, you might not see that due to AQE has not finalized the plan. Or go to the Spark UI and click on the application you ran and on the "SQL" tab. If you click the operation "count at ...", you should see the graph of Spark -Executors and some of those should have the "GPU" label as well. \ No newline at end of file +Executors and some of those should have the "GPU" label as well. + +If you want to monitor GPU metrics on Dataproc, you can create the cluster with additional +[metadata](https://cloud.google.com/dataproc/docs/concepts/configuring-clusters/metadata) and +[scopes](https://cloud.google.com/sdk/gcloud/reference/dataproc/clusters/create#--scopes): +``` +--metadata install-gpu-agent="true" +--scopes monitoring +``` +You can then monitor the following metrics on [Web UI](https://console.cloud.google.com/monitoring/metrics-explorer), +we should be able to see "Resource & Metric" -> "VM Instance" -> "Custom": +* **custom.googleapis.com/instance/gpu/utilization** - The GPU cores utilization in %. +* **custom.googleapis.com/instance/gpu/memory_utilization** - The GPU memory bandwidth utilization in %. +* **custom.googleapis.com/instance/gpu/memory_total** - Total memory of the GPU card in MB. +* **custom.googleapis.com/instance/gpu/memory_used** - Used memory of the GPU card. +* **custom.googleapis.com/instance/gpu/memory_free** - Available memory of the GPU card. +* **custom.googleapis.com/instance/gpu/temperature** - Temperature of the GPU. +The metrics are sent with attached label, marking them by the gpu_type and gpu_bus_id. +This way, instances with multiple GPUs attached can report the metrics of their cards separately. +You can later aggregate or filter those metrics in the Cloud Monitoring systems. diff --git a/sparkRapids/__init__.py b/spark-rapids/__init__.py similarity index 100% rename from sparkRapids/__init__.py rename to spark-rapids/__init__.py diff --git a/sparkRapids/spark-rapids.sh b/spark-rapids/spark-rapids.sh similarity index 78% rename from sparkRapids/spark-rapids.sh rename to spark-rapids/spark-rapids.sh index 948bb645c..b5b3d0d74 100644 --- a/sparkRapids/spark-rapids.sh +++ b/spark-rapids/spark-rapids.sh @@ -12,11 +12,11 @@ OS_NAME=$(lsb_release -is | tr '[:upper:]' '[:lower:]') readonly OS_NAME readonly SPARK_VERSION_ENV=$(spark-submit --version 2>&1 | sed -n 's/.*version[[:blank:]]\+\([0-9]\+\.[0-9]\).*/\1/p' | head -n1) -readonly DEFAULT_SPARK_RAPIDS_VERSION="22.08.0" +readonly DEFAULT_SPARK_RAPIDS_VERSION="22.10.0" if [[ "${SPARK_VERSION_ENV}" == "3"* ]]; then readonly DEFAULT_CUDA_VERSION="11.5" - readonly DEFAULT_CUDF_VERSION="22.08.0" + readonly DEFAULT_CUDF_VERSION="22.10.0" readonly DEFAULT_XGBOOST_VERSION="1.6.2" readonly DEFAULT_XGBOOST_GPU_SUB_VERSION="0.3.0" readonly SPARK_VERSION="3.0" @@ -35,9 +35,10 @@ readonly RUNTIME=$(get_metadata_attribute 'rapids-runtime' 'SPARK') # CUDA version and Driver version config CUDA_VERSION=$(get_metadata_attribute 'cuda-version' '11.5') -readonly DEFAULT_NVIDIA_DEBIAN_GPU_DRIVER_VERSION='495.29.05' +DEFAULT_NVIDIA_DEBIAN_GPU_DRIVER_VERSION=$(get_metadata_attribute 'driver-version' '495.29.05') readonly CUDA_VERSION +readonly DEFAULT_NVIDIA_DEBIAN_GPU_DRIVER_VERSION readonly DEFAULT_NVIDIA_DEBIAN_GPU_DRIVER_VERSION_PREFIX=${DEFAULT_NVIDIA_DEBIAN_GPU_DRIVER_VERSION%%.*} # Parameters for NVIDIA-provided Debian GPU driver @@ -108,7 +109,7 @@ function install_spark_rapids() { local -r rapids_repo_url='https://repo1.maven.org/maven2/ai/rapids' local -r nvidia_repo_url='https://repo1.maven.org/maven2/com/nvidia' local -r dmlc_repo_url='https://repo.maven.apache.org/maven2/ml/dmlc' - + # Convert . to - for URL formatting local cudf_cuda_version="${CUDA_VERSION//\./-}" @@ -143,7 +144,7 @@ function configure_spark() { cat >>${SPARK_CONF_DIR}/spark-defaults.conf </lib/systemd/system/gpu-utilization-agent.service -[Unit] -Description=GPU Utilization Metric Agent + mkdir -p /opt/google + chmod 777 /opt/google + cd /opt/google + execute_with_retries "git clone https://github.com/GoogleCloudPlatform/compute-gpu-monitoring.git" +} -[Service] -Type=simple -PIDFile=/run/gpu_agent.pid -ExecStart=/bin/bash --login -c 'python "${install_dir}/report_gpu_metrics.py"' -User=root -Group=root -WorkingDirectory=/ -Restart=always +function install_agent_dependency(){ + cd /opt/google/compute-gpu-monitoring/linux + python3 -m venv venv + venv/bin/pip install wheel + venv/bin/pip install -Ur requirements.txt +} -[Install] -WantedBy=multi-user.target -EOF - # Reload systemd manager configuration +function start_agent_service(){ + cp /opt/google/compute-gpu-monitoring/linux/systemd/google_gpu_monitoring_agent_venv.service /lib/systemd/system systemctl daemon-reload - # Enable gpu-utilization-agent service - systemctl --no-reload --now enable gpu-utilization-agent.service + systemctl --no-reload --now enable /lib/systemd/system/google_gpu_monitoring_agent_venv.service } function set_hadoop_property() { @@ -307,10 +305,10 @@ function configure_gpu_exclusive_mode() { function fetch_mig_scripts() { mkdir -p /usr/local/yarn-mig-scripts - sudo chmod 755 /usr/local/yarn-mig-scripts - wget -P /usr/local/yarn-mig-scripts/ https://raw.githubusercontent.com/NVIDIA/spark-rapids-examples/branch-22.08/examples/MIG-Support/yarn-unpatched/scripts/nvidia-smi - wget -P /usr/local/yarn-mig-scripts/ https://raw.githubusercontent.com/NVIDIA/spark-rapids-examples/branch-22.08/examples/MIG-Support/yarn-unpatched/scripts/mig2gpu.sh - sudo chmod 755 /usr/local/yarn-mig-scripts/* + chmod 755 /usr/local/yarn-mig-scripts + wget -P /usr/local/yarn-mig-scripts/ https://raw.githubusercontent.com/NVIDIA/spark-rapids-examples/branch-22.10/examples/MIG-Support/yarn-unpatched/scripts/nvidia-smi + wget -P /usr/local/yarn-mig-scripts/ https://raw.githubusercontent.com/NVIDIA/spark-rapids-examples/branch-22.10/examples/MIG-Support/yarn-unpatched/scripts/mig2gpu.sh + chmod 755 /usr/local/yarn-mig-scripts/* } function configure_gpu_script() { @@ -380,20 +378,16 @@ EOF } function setup_gpu_yarn() { - if [[ ${OS_NAME} != debian ]] && [[ ${OS_NAME} != ubuntu ]] && [[ ${OS_NAME} != rocky ]]; then - echo "Unsupported OS: '${OS_NAME}'" - exit 1 - fi if [[ ${OS_NAME} == debian ]] || [[ ${OS_NAME} == ubuntu ]]; then export DEBIAN_FRONTEND=noninteractive execute_with_retries "apt-get update" execute_with_retries "apt-get install -y -q pciutils" elif [[ ${OS_NAME} == rocky ]] ; then - execute_with_retries "dnf -y -q update" execute_with_retries "dnf -y -q install pciutils" - execute_with_retries "dnf -y -q install kernel-devel" - execute_with_retries "dnf -y -q install gcc" + else + echo "Unsupported OS: '${OS_NAME}'" + exit 1 fi # This configuration should be ran on all nodes @@ -417,6 +411,8 @@ function setup_gpu_yarn() { if [[ ${OS_NAME} == debian ]] || [[ ${OS_NAME} == ubuntu ]]; then execute_with_retries "apt-get install -y -q 'linux-headers-$(uname -r)'" + elif [[ ${OS_NAME} == rocky ]]; then + echo "kernel devel and headers not required on rocky. installing from binary" fi # if mig is enabled drivers would have already been installed @@ -442,16 +438,91 @@ function setup_gpu_yarn() { fi # Restart YARN services if they are running already - if [[ $(systemctl show hadoop-yarn-resourcemanager.service -p SubState --value) == 'running' ]]; then - systemctl restart hadoop-yarn-resourcemanager.service + for svc in resourcemanager nodemanager; do + if [[ $(systemctl show hadoop-yarn-${svc}.service -p SubState --value) == 'running' ]]; then + systemctl restart hadoop-yarn-${svc}.service + fi + done +} + +function upgrade_kernel() { + # Determine which kernel is installed + if [[ "${OS_NAME}" == "debian" ]]; then + CURRENT_KERNEL_VERSION=`cat /proc/version | perl -ne 'print( / Debian (\S+) / )'` + elif [[ "${OS_NAME}" == "ubuntu" ]]; then + CURRENT_KERNEL_VERSION=`cat /proc/version | perl -ne 'print( /^Linux version (\S+) / )'` + elif [[ ${OS_NAME} == rocky ]]; then + KERN_VER=$(yum info --installed kernel | awk '/^Version/ {print $3}') + KERN_REL=$(yum info --installed kernel | awk '/^Release/ {print $3}') + CURRENT_KERNEL_VERSION="${KERN_VER}-${KERN_REL}" + else + echo "unsupported OS: ${OS_NAME}!" + exit -1 + fi + + # Get latest version available in repos + if [[ "${OS_NAME}" == "debian" ]]; then + apt-get -qq update + TARGET_VERSION=$(apt-cache show --no-all-versions linux-image-amd64 | awk '/^Version/ {print $2}') + elif [[ "${OS_NAME}" == "ubuntu" ]]; then + apt-get -qq update + LATEST_VERSION=$(apt-cache show --no-all-versions linux-image-gcp | awk '/^Version/ {print $2}') + TARGET_VERSION=`echo ${LATEST_VERSION} | perl -ne 'printf(q{%s-%s-gcp},/(\d+\.\d+\.\d+)\.(\d+)/)'` + elif [[ "${OS_NAME}" == "rocky" ]]; then + if yum info --available kernel ; then + KERN_VER=$(yum info --available kernel | awk '/^Version/ {print $3}') + KERN_REL=$(yum info --available kernel | awk '/^Release/ {print $3}') + TARGET_VERSION="${KERN_VER}-${KERN_REL}" + else + TARGET_VERSION="${CURRENT_KERNEL_VERSION}" + fi + fi + + # Skip this script if we are already on the target version + if [[ "${CURRENT_KERNEL_VERSION}" == "${TARGET_VERSION}" ]]; then + echo "target kernel version [${TARGET_VERSION}] is installed" + + # Reboot may have interrupted dpkg. Bring package system to a good state + if [[ "${OS_NAME}" == "debian" || "${OS_NAME}" == "ubuntu" ]]; then + dpkg --configure -a + fi + + return 0 fi - if [[ $(systemctl show hadoop-yarn-nodemanager.service -p SubState --value) == 'running' ]]; then - systemctl restart hadoop-yarn-nodemanager.service + + # Install the latest kernel + if [[ ${OS_NAME} == debian ]]; then + apt-get install -y linux-image-amd64 + elif [[ "${OS_NAME}" == "ubuntu" ]]; then + apt-get install -y linux-image-gcp + elif [[ "${OS_NAME}" == "rocky" ]]; then + dnf -y -q install kernel fi + + # Make it possible to reboot before init actions are complete - #1033 + DP_ROOT=/usr/local/share/google/dataproc + STARTUP_SCRIPT="${DP_ROOT}/startup-script.sh" + POST_HDFS_STARTUP_SCRIPT="${DP_ROOT}/post-hdfs-startup-script.sh" + + for startup_script in ${STARTUP_SCRIPT} ${POST_HDFS_STARTUP_SCRIPT} ; do + sed -i -e 's:/usr/bin/env bash:/usr/bin/env bash\nexit 0:' ${startup_script} + done + + cp /var/log/dataproc-initialization-script-0.log /var/log/dataproc-initialization-script-0.log.0 + + systemctl reboot } + function main() { + + if [[ "${OS_NAME}" == "rocky" ]]; then + if dnf list kernel-devel-$(uname -r) && list kernel-headers-$(uname -r); then + echo "kernel devel and headers packages are available. Proceed without kernel upgrade." + else + upgrade_kernel + fi + fi setup_gpu_yarn - if [[ "${RUNTIME}" == "SPARK" ]]; then install_spark_rapids configure_spark @@ -459,17 +530,13 @@ function main() { else echo "Unsupported RAPIDS Runtime: ${RUNTIME}" exit 1 - fi + fi - if [[ "${ROLE}" == "Master" ]]; then - systemctl restart hadoop-yarn-resourcemanager.service - # Restart NodeManager on Master as well if this is a single-node-cluster. - if systemctl status hadoop-yarn-nodemanager; then - systemctl restart hadoop-yarn-nodemanager.service + for svc in resourcemanager nodemanager; do + if [[ $(systemctl show hadoop-yarn-${svc}.service -p SubState --value) == 'running' ]]; then + systemctl restart hadoop-yarn-${svc}.service fi - else - systemctl restart hadoop-yarn-nodemanager.service - fi + done } main diff --git a/sparkRapids/test_sparkRapids.py b/spark-rapids/test_spark_rapids.py similarity index 79% rename from sparkRapids/test_sparkRapids.py rename to spark-rapids/test_spark_rapids.py index fb62f8773..a37917812 100644 --- a/sparkRapids/test_sparkRapids.py +++ b/spark-rapids/test_spark_rapids.py @@ -6,11 +6,11 @@ from integration_tests.dataproc_test_case import DataprocTestCase -class RapidsTestCase(DataprocTestCase): +class SparkRapidsTestCase(DataprocTestCase): COMPONENT = "rapids" - INIT_ACTIONS = ["sparkRapids/spark-rapids.sh"] + INIT_ACTIONS = ["spark-rapids/spark-rapids.sh"] - GPU_P100 = "type=nvidia-tesla-p100" + GPU_T4 = "type=nvidia-tesla-t4" # Tests for RAPIDS init action XGBOOST_SPARK_TEST_SCRIPT_FILE_NAME = "verify_xgboost_spark_rapids.scala" @@ -33,17 +33,15 @@ def verify_spark_job(self): self.remove_test_script(self.XGBOOST_SPARK_TEST_SCRIPT_FILE_NAME, instance_name) - @parameterized.parameters(("SINGLE", ["m"], GPU_P100), - ("STANDARD", ["w-0"], GPU_P100)) - def test_rapids_spark(self, configuration, machine_suffixes, accelerator): - if self.getImageOs() == "rocky": - self.skipTest("Not supported in Rocky Linux-based images") + @parameterized.parameters(("SINGLE", ["m"], GPU_T4), + ("STANDARD", ["w-0"], GPU_T4)) + def test_spark_rapids(self, configuration, machine_suffixes, accelerator): + + if self.getImageVersion() < pkg_resources.parse_version("2.0") and self.getImageOs() == "rocky": + self.skipTest("Not supported in pre 2.0 rocky images") optional_components = None metadata = "gpu-driver-provider=NVIDIA,rapids-runtime=SPARK" - if self.getImageVersion() < pkg_resources.parse_version("2.0"): - optional_components = ["ANACONDA"] - metadata += ",cuda-version=10.1" self.createCluster( configuration, @@ -53,6 +51,7 @@ def test_rapids_spark(self, configuration, machine_suffixes, accelerator): machine_type="n1-standard-4", master_accelerator=accelerator if configuration == "SINGLE" else None, worker_accelerator=accelerator, + boot_disk_size="1024GB", timeout_in_minutes=30) for machine_suffix in machine_suffixes: @@ -61,14 +60,12 @@ def test_rapids_spark(self, configuration, machine_suffixes, accelerator): # Only need to do this once self.verify_spark_job() - @parameterized.parameters(("STANDARD", ["w-0"], GPU_P100, "11.2")) + @parameterized.parameters(("STANDARD", ["w-0"], GPU_T4, "11.2")) def test_non_default_cuda_versions(self, configuration, machine_suffixes, accelerator, cuda_version): - if self.getImageOs() == "rocky": - self.skipTest("Not supported in Rocky Linux-based images") - if self.getImageVersion() < pkg_resources.parse_version("2.0"): - self.skipTest("Not supported in pre 2.0 images") + if self.getImageVersion() < pkg_resources.parse_version("2.0") and self.getImageOs() == "rocky": + self.skipTest("Not supported in pre 2.0 rocky images") metadata = ("gpu-driver-provider=NVIDIA,rapids-runtime=SPARK" ",cuda-version={}".format(cuda_version)) @@ -80,6 +77,7 @@ def test_non_default_cuda_versions(self, configuration, machine_suffixes, machine_type="n1-standard-4", master_accelerator=accelerator if configuration == "SINGLE" else None, worker_accelerator=accelerator, + boot_disk_size="1024GB", timeout_in_minutes=30) for machine_suffix in machine_suffixes: diff --git a/sparkRapids/verify_xgboost_spark_rapids.scala b/spark-rapids/verify_xgboost_spark_rapids.scala similarity index 96% rename from sparkRapids/verify_xgboost_spark_rapids.scala rename to spark-rapids/verify_xgboost_spark_rapids.scala index 2d4db5aae..5396209fc 100644 --- a/sparkRapids/verify_xgboost_spark_rapids.scala +++ b/spark-rapids/verify_xgboost_spark_rapids.scala @@ -11,7 +11,7 @@ val schema = StructType(Array( val features = schema.filter(_.name != label).map(_.name).toArray -val datas = Seq(Row(1.05, 9.05, 0), +val datas = Seq(Row(1.05, 9.05, 0), Row(2.95, 1.95, 1)) val df = spark.createDataFrame( spark.sparkContext.parallelize(datas),schema)