From e63e026237048f80e172ab6323bfacce6910cd81 Mon Sep 17 00:00:00 2001 From: Furcy Pin Date: Tue, 12 Apr 2022 12:07:58 +0200 Subject: [PATCH 1/4] [SPARK-38870][SQL][PYSPARK] Make SparkSession.builder return new builder ## What changes were proposed in this pull request? In Scala, SparkSession.builder returns a new builder each time, but in Python, it returns the same builder every time, with the same static options. Because of this, whenever SparkSession.builder.getOrCreate() is called, the options set in the very first builder are reapplied. To fix this, I introduce a @classproperty decorator, that is similar to the @property decorator, except that it works with class methods. Now, whenever `SparkSession.builder` is called, a new builder is returned as expected. I also made the parameter `Builder._options` non-static, and I removed `Builder._sc` which didn't seem used anymore. `Builder._sc` was introduced in this commit: 58419b92673c46911c25bc6c6b13397f880c6424 And it's only use seems to have been removed by this commit: 88696ebcb72fd3057b1546831f653b29b7e0abb2 ## How was this patch tested? I updated the Doctests for SparkSession.Builder.getOrCreate and added Doctests on the classproperty decorator as well. --- python/pyspark/sql/session.py | 50 ++++++++++++++++++++++++++++++----- 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 8f4809907b599..3b835304b9d37 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -104,6 +104,35 @@ def toDF(self, schema=None, sampleRatio=None): RDD.toDF = toDF # type: ignore[assignment] +class classproperty(property): + """Same as Python's @property annotation, but for class attributes. + + Example: + + >>> class Builder: + ... + ... def build(self): + ... return MyClass() + ... + >>> class MyClass: + ... + ... @classproperty + ... def builder(cls): + ... print("instantiating new builder") + ... return Builder() + >>> c1 = MyClass.builder + instantiating new builder + >>> c2 = MyClass.builder + instantiating new builder + >>> c1 == c2 + False + >>> isinstance(c1.build(), MyClass) + True + """ + def __get__(self, cls, owner): + return classmethod(self.fget).__get__(None, owner)() + + class SparkSession(SparkConversionMixin): """The entry point to programming Spark with the Dataset and DataFrame API. @@ -142,8 +171,9 @@ class Builder: """Builder for :class:`SparkSession`.""" _lock = RLock() - _options: Dict[str, Any] = {} - _sc: Optional[SparkContext] = None + + def __init__(self): + self._options: Dict[str, Any] = {} @overload def config(self, *, conf: SparkConf) -> "SparkSession.Builder": @@ -247,13 +277,19 @@ def getOrCreate(self) -> "SparkSession": >>> s1.conf.get("k1") == "v1" True + The configuration of the SparkSession can be changed afterwards + + >>> s1.conf.set("k1", "v1_new") + >>> s1.conf.get("k1") == "v1_new" + True + In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. >>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() - >>> s1.conf.get("k1") == s2.conf.get("k1") + >>> s1.conf.get("k1") == s2.conf.get("k1") == "v1_new" True - >>> s1.conf.get("k2") == s2.conf.get("k2") + >>> s1.conf.get("k2") == s2.conf.get("k2") == "v2" True """ with self._lock: @@ -276,8 +312,10 @@ def getOrCreate(self) -> "SparkSession": ).applyModifiableSettings(session._jsparkSession, self._options) return session - builder = Builder() - """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances.""" + @classproperty + def builder(cls): + """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances.""" + return cls.Builder() _instantiatedSession: ClassVar[Optional["SparkSession"]] = None _activeSession: ClassVar[Optional["SparkSession"]] = None From 60a7975fd0b7f5154bc9a177d0b4a54118c0105d Mon Sep 17 00:00:00 2001 From: Furcy Pin Date: Tue, 12 Apr 2022 12:07:58 +0200 Subject: [PATCH 2/4] [SPARK-38870][SQL][PYSPARK] Make SparkSession.builder return new builder ## Update of previous commit Since Python 3.9, the @classmethod decorator is compatible with the @property decorator. This change adds a check on the Python version to use the "proper" way in 3.9 and beyond, and a TODO to remember cleaning up once support for Python 3.8 is dropped. --- python/pyspark/sql/session.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 3b835304b9d37..6783710a18896 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -104,8 +104,14 @@ def toDF(self, schema=None, sampleRatio=None): RDD.toDF = toDF # type: ignore[assignment] +# TODO(SPARK-38912): This method can be dropped once support for Python 3.8 is dropped +# In Python 3.9, the @property decorator has been made compatible with the @classmethod decorator. +# https://docs.python.org/3.9/library/functions.html#classmethod +# +# @classmethod + @property is also affected by a bug in Python's docstring which was backported +# to Python 3.9.6 (https://github.com/python/cpython/pull/28838) class classproperty(property): - """Same as Python's @property annotation, but for class attributes. + """Same as Python's @property decorator, but for class attributes. Example: @@ -129,6 +135,7 @@ class classproperty(property): >>> isinstance(c1.build(), MyClass) True """ + def __get__(self, cls, owner): return classmethod(self.fget).__get__(None, owner)() @@ -312,6 +319,14 @@ def getOrCreate(self) -> "SparkSession": ).applyModifiableSettings(session._jsparkSession, self._options) return session + # TODO(SPARK-38912): Replace @classproperty with @classmethod + @property once support for + # Python 3.8 is dropped. + # + # In Python 3.9, the @property decorator has been made compatible with the @classmethod decorator. + # https://docs.python.org/3.9/library/functions.html#classmethod + # + # @classmethod + @property is also affected by a bug in Python's docstring which was backported + # to Python 3.9.6 (https://github.com/python/cpython/pull/28838) @classproperty def builder(cls): """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances.""" From 8f5d5cda499668c6e48958cacdab5aafb3d5c387 Mon Sep 17 00:00:00 2001 From: Furcy Pin Date: Wed, 20 Apr 2022 16:32:48 +0200 Subject: [PATCH 3/4] [SPARK-38870][SQL][PYSPARK] Make SparkSession.builder return new builder ## Update of previous commits Update docstring for SparkSession.builder, to make it match with Scala's description. --- python/pyspark/sql/session.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 6783710a18896..9bcf1f20df18f 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -105,8 +105,8 @@ def toDF(self, schema=None, sampleRatio=None): # TODO(SPARK-38912): This method can be dropped once support for Python 3.8 is dropped -# In Python 3.9, the @property decorator has been made compatible with the @classmethod decorator. -# https://docs.python.org/3.9/library/functions.html#classmethod +# In Python 3.9, the @property decorator has been made compatible with the +# @classmethod decorator (https://docs.python.org/3.9/library/functions.html#classmethod) # # @classmethod + @property is also affected by a bug in Python's docstring which was backported # to Python 3.9.6 (https://github.com/python/cpython/pull/28838) @@ -322,14 +322,14 @@ def getOrCreate(self) -> "SparkSession": # TODO(SPARK-38912): Replace @classproperty with @classmethod + @property once support for # Python 3.8 is dropped. # - # In Python 3.9, the @property decorator has been made compatible with the @classmethod decorator. - # https://docs.python.org/3.9/library/functions.html#classmethod + # In Python 3.9, the @property decorator has been made compatible with the + # @classmethod decorator (https://docs.python.org/3.9/library/functions.html#classmethod) # # @classmethod + @property is also affected by a bug in Python's docstring which was backported # to Python 3.9.6 (https://github.com/python/cpython/pull/28838) @classproperty def builder(cls): - """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances.""" + """Creates a :class:`Builder` for constructing a :class:`SparkSession`.""" return cls.Builder() _instantiatedSession: ClassVar[Optional["SparkSession"]] = None From 79323e8129ba2f362534440a715ca579d99e5d32 Mon Sep 17 00:00:00 2001 From: Furcy Pin Date: Thu, 21 Apr 2022 18:35:05 +0200 Subject: [PATCH 4/4] [SPARK-38870][SQL][PYSPARK] Make SparkSession.builder return new builder ## Update of previous commits Fix linter's warnings For some obscure reason, the CI linting step fails with the following error: starting mypy examples test... examples failed mypy checks: examples/src/main/python/pagerank.py:79: error: unused "type: ignore" comment This is extremely weird as the pagerank.py seems to be completely unrelated to the changes made in this ticket. We removed this "type: ignore" comment to make mypy happy. --- python/pyspark/sql/session.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 9bcf1f20df18f..688d67d10d320 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -136,8 +136,12 @@ class classproperty(property): True """ - def __get__(self, cls, owner): - return classmethod(self.fget).__get__(None, owner)() + def __get__(self, instance: Any, owner: Any = None) -> "SparkSession.Builder": + # The "type: ignore" below silences the following error from mypy: + # error: Argument 1 to "classmethod" has incompatible + # type "Optional[Callable[[Any], Any]]"; + # expected "Callable[..., Any]" [arg-type] + return classmethod(self.fget).__get__(None, owner)() # type: ignore class SparkSession(SparkConversionMixin): @@ -179,7 +183,7 @@ class Builder: _lock = RLock() - def __init__(self): + def __init__(self) -> None: self._options: Dict[str, Any] = {} @overload @@ -328,7 +332,7 @@ def getOrCreate(self) -> "SparkSession": # @classmethod + @property is also affected by a bug in Python's docstring which was backported # to Python 3.9.6 (https://github.com/python/cpython/pull/28838) @classproperty - def builder(cls): + def builder(cls) -> Builder: """Creates a :class:`Builder` for constructing a :class:`SparkSession`.""" return cls.Builder()