From 39acf9901644027aa1cbc0a161afdc44b1f9eae6 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Tue, 14 Dec 2021 16:14:19 +0900 Subject: [PATCH 1/2] Use existing active Spark session instead of SparkSession.getOrCreate in pandas API on Spark --- python/pyspark/pandas/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/python/pyspark/pandas/utils.py b/python/pyspark/pandas/utils.py index be71d70ffc14e..267ab463b6f33 100644 --- a/python/pyspark/pandas/utils.py +++ b/python/pyspark/pandas/utils.py @@ -468,6 +468,10 @@ def default_session(conf: Optional[Dict[str, Any]] = None) -> SparkSession: if conf is None: conf = dict() + spark = SparkSession.getActiveSession() + if spark is not None: + return spark + builder = SparkSession.builder.appName("pandas-on-Spark") for key, value in conf.items(): builder = builder.config(key, value) From ce0cd8be5340b6b8804c50ffbe2ec92036ece341 Mon Sep 17 00:00:00 2001 From: Hyukjin Kwon Date: Tue, 14 Dec 2021 19:05:26 +0900 Subject: [PATCH 2/2] Remove unused confs --- python/pyspark/pandas/utils.py | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/python/pyspark/pandas/utils.py b/python/pyspark/pandas/utils.py index 267ab463b6f33..9c15ea3a6f01f 100644 --- a/python/pyspark/pandas/utils.py +++ b/python/pyspark/pandas/utils.py @@ -464,24 +464,12 @@ def is_testing() -> bool: return "SPARK_TESTING" in os.environ -def default_session(conf: Optional[Dict[str, Any]] = None) -> SparkSession: - if conf is None: - conf = dict() - +def default_session() -> SparkSession: spark = SparkSession.getActiveSession() if spark is not None: return spark builder = SparkSession.builder.appName("pandas-on-Spark") - for key, value in conf.items(): - builder = builder.config(key, value) - # Currently, pandas-on-Spark is dependent on such join due to 'compute.ops_on_diff_frames' - # configuration. This is needed with Spark 3.0+. - builder.config("spark.sql.analyzer.failAmbiguousSelfJoin", False) - - if is_testing(): - builder.config("spark.executor.allowSparkContext", False) - return builder.getOrCreate()