From f01b3847644325ab3e1c0d85cd58b5934dd9bc20 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 15 Nov 2019 21:17:36 -0800 Subject: [PATCH 1/2] JDK11 pyspark test --- python/run-tests.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/run-tests.py b/python/run-tests.py index b1119b044d71f..d7e690c4c663a 100755 --- a/python/run-tests.py +++ b/python/run-tests.py @@ -88,8 +88,8 @@ def run_individual_python_test(target_dir, test_name, pyspark_python): # Also override the JVM's temp directory by setting driver and executor options. spark_args = [ - "--conf", "spark.driver.extraJavaOptions=-Djava.io.tmpdir={0}".format(tmp_dir), - "--conf", "spark.executor.extraJavaOptions=-Djava.io.tmpdir={0}".format(tmp_dir), + "--conf", "spark.driver.extraJavaOptions='-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true'".format(tmp_dir), + "--conf", "spark.executor.extraJavaOptions='-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true'".format(tmp_dir), "pyspark-shell" ] env["PYSPARK_SUBMIT_ARGS"] = " ".join(spark_args) From e4c1fc56e3b11fe99d0e2c826df20d07278fd65c Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Fri, 15 Nov 2019 21:31:51 -0800 Subject: [PATCH 2/2] Fix python style --- python/run-tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/run-tests.py b/python/run-tests.py index d7e690c4c663a..282fe5afa1963 100755 --- a/python/run-tests.py +++ b/python/run-tests.py @@ -87,9 +87,10 @@ def run_individual_python_test(target_dir, test_name, pyspark_python): env["TMPDIR"] = tmp_dir # Also override the JVM's temp directory by setting driver and executor options. + java_options = "-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true".format(tmp_dir) spark_args = [ - "--conf", "spark.driver.extraJavaOptions='-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true'".format(tmp_dir), - "--conf", "spark.executor.extraJavaOptions='-Djava.io.tmpdir={0} -Dio.netty.tryReflectionSetAccessible=true'".format(tmp_dir), + "--conf", "spark.driver.extraJavaOptions='{0}'".format(java_options), + "--conf", "spark.executor.extraJavaOptions='{0}'".format(java_options), "pyspark-shell" ] env["PYSPARK_SUBMIT_ARGS"] = " ".join(spark_args)