diff --git a/python/pyspark/sql/session.py b/python/pyspark/sql/session.py index 8f4809907b599..688d67d10d320 100644 --- a/python/pyspark/sql/session.py +++ b/python/pyspark/sql/session.py @@ -104,6 +104,46 @@ def toDF(self, schema=None, sampleRatio=None): RDD.toDF = toDF # type: ignore[assignment] +# TODO(SPARK-38912): This method can be dropped once support for Python 3.8 is dropped +# In Python 3.9, the @property decorator has been made compatible with the +# @classmethod decorator (https://docs.python.org/3.9/library/functions.html#classmethod) +# +# @classmethod + @property is also affected by a bug in Python's docstring which was backported +# to Python 3.9.6 (https://github.com/python/cpython/pull/28838) +class classproperty(property): + """Same as Python's @property decorator, but for class attributes. + + Example: + + >>> class Builder: + ... + ... def build(self): + ... return MyClass() + ... + >>> class MyClass: + ... + ... @classproperty + ... def builder(cls): + ... print("instantiating new builder") + ... return Builder() + >>> c1 = MyClass.builder + instantiating new builder + >>> c2 = MyClass.builder + instantiating new builder + >>> c1 == c2 + False + >>> isinstance(c1.build(), MyClass) + True + """ + + def __get__(self, instance: Any, owner: Any = None) -> "SparkSession.Builder": + # The "type: ignore" below silences the following error from mypy: + # error: Argument 1 to "classmethod" has incompatible + # type "Optional[Callable[[Any], Any]]"; + # expected "Callable[..., Any]" [arg-type] + return classmethod(self.fget).__get__(None, owner)() # type: ignore + + class SparkSession(SparkConversionMixin): """The entry point to programming Spark with the Dataset and DataFrame API. @@ -142,8 +182,9 @@ class Builder: """Builder for :class:`SparkSession`.""" _lock = RLock() - _options: Dict[str, Any] = {} - _sc: Optional[SparkContext] = None + + def __init__(self) -> None: + self._options: Dict[str, Any] = {} @overload def config(self, *, conf: SparkConf) -> "SparkSession.Builder": @@ -247,13 +288,19 @@ def getOrCreate(self) -> "SparkSession": >>> s1.conf.get("k1") == "v1" True + The configuration of the SparkSession can be changed afterwards + + >>> s1.conf.set("k1", "v1_new") + >>> s1.conf.get("k1") == "v1_new" + True + In case an existing SparkSession is returned, the config options specified in this builder will be applied to the existing SparkSession. >>> s2 = SparkSession.builder.config("k2", "v2").getOrCreate() - >>> s1.conf.get("k1") == s2.conf.get("k1") + >>> s1.conf.get("k1") == s2.conf.get("k1") == "v1_new" True - >>> s1.conf.get("k2") == s2.conf.get("k2") + >>> s1.conf.get("k2") == s2.conf.get("k2") == "v2" True """ with self._lock: @@ -276,8 +323,18 @@ def getOrCreate(self) -> "SparkSession": ).applyModifiableSettings(session._jsparkSession, self._options) return session - builder = Builder() - """A class attribute having a :class:`Builder` to construct :class:`SparkSession` instances.""" + # TODO(SPARK-38912): Replace @classproperty with @classmethod + @property once support for + # Python 3.8 is dropped. + # + # In Python 3.9, the @property decorator has been made compatible with the + # @classmethod decorator (https://docs.python.org/3.9/library/functions.html#classmethod) + # + # @classmethod + @property is also affected by a bug in Python's docstring which was backported + # to Python 3.9.6 (https://github.com/python/cpython/pull/28838) + @classproperty + def builder(cls) -> Builder: + """Creates a :class:`Builder` for constructing a :class:`SparkSession`.""" + return cls.Builder() _instantiatedSession: ClassVar[Optional["SparkSession"]] = None _activeSession: ClassVar[Optional["SparkSession"]] = None