From 0b6de3aaa075e9d1c653f29acb0d33e7c7b8f018 Mon Sep 17 00:00:00 2001 From: Uros Bojanic Date: Fri, 28 Nov 2025 22:08:17 +0100 Subject: [PATCH 1/6] Geography --- .../sql/GeographyConnectDataFrameSuite.scala | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala index 2016a84ac5a35..5accc0b843d22 100644 --- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala +++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala @@ -103,4 +103,27 @@ class GeographyConnectDataFrameSuite extends QueryTest with RemoteSparkSession { val expectedGeog = Geography.fromWKB(point, 4326) checkAnswer(df, Seq(Row(expectedGeog))) } + + test("geospatial feature disabled") { + withSQLConf("spark.sql.geospatial.enabled" -> "false") { + val geography = Geography.fromWKB(point1, 4326) + val schema = StructType(Seq(StructField("col1", GeographyType(4326)))) + // Java List[Row] + schema. + val javaList = java.util.Arrays.asList(Row(geography)) + checkError( + exception = intercept[AnalysisException] { + spark.createDataFrame(javaList, schema).collect() + }, + condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED" + ) + // Implicit encoder path. + import testImplicits._ + checkError( + exception = intercept[AnalysisException] { + Seq(geography).toDF("g").collect() + }, + condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED" + ) + } + } } From 24c4bcd599ae031bbf003ae58a7720acf571c3fc Mon Sep 17 00:00:00 2001 From: Uros Bojanic Date: Fri, 28 Nov 2025 23:23:02 +0100 Subject: [PATCH 2/6] Fix scalafmt --- .../apache/spark/sql/GeographyConnectDataFrameSuite.scala | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala index 5accc0b843d22..f3f52366df666 100644 --- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala +++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeographyConnectDataFrameSuite.scala @@ -114,16 +114,14 @@ class GeographyConnectDataFrameSuite extends QueryTest with RemoteSparkSession { exception = intercept[AnalysisException] { spark.createDataFrame(javaList, schema).collect() }, - condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED" - ) + condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED") // Implicit encoder path. import testImplicits._ checkError( exception = intercept[AnalysisException] { Seq(geography).toDF("g").collect() }, - condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED" - ) + condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED") } } } From 313681a57ede3cff9e14ee0646c3bc82cbbc1afa Mon Sep 17 00:00:00 2001 From: Uros Bojanic Date: Wed, 3 Dec 2025 15:50:38 +0100 Subject: [PATCH 3/6] Unsupported geospatial --- .../sql/connect/execution/SparkConnectPlanExecution.scala | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala index f5cb2696d849b..bb26e965f7151 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala @@ -29,6 +29,7 @@ import org.apache.spark.connect.proto import org.apache.spark.connect.proto.ExecutePlanResponse import org.apache.spark.sql.Row import org.apache.spark.sql.catalyst.InternalRow +import org.apache.spark.sql.catalyst.expressions.st.STExpressionUtils import org.apache.spark.sql.classic.{DataFrame, Dataset} import org.apache.spark.sql.connect.common.DataTypeProtoConverter import org.apache.spark.sql.connect.common.LiteralValueProtoConverter.toLiteralProto @@ -126,6 +127,13 @@ private[execution] class SparkConnectPlanExecution(executeHolder: ExecuteHolder) val sessionId = executePlan.sessionHolder.sessionId val spark = dataframe.sparkSession val schema = dataframe.schema + val geospatialEnabled = spark.sessionState.conf.geospatialEnabled + if (!geospatialEnabled && schema.fields.exists(field => + STExpressionUtils.isGeoSpatialType(field.dataType))) { + throw new org.apache.spark.sql.AnalysisException( + errorClass = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED", + messageParameters = scala.collection.immutable.Map.empty) + } val maxRecordsPerBatch = spark.sessionState.conf.arrowMaxRecordsPerBatch val timeZoneId = spark.sessionState.conf.sessionLocalTimeZone val largeVarTypes = spark.sessionState.conf.arrowUseLargeVarTypes From 6dd399ae6baf53d8eb205f19e34e126cb2cdf060 Mon Sep 17 00:00:00 2001 From: Uros Bojanic Date: Wed, 3 Dec 2025 18:19:22 +0100 Subject: [PATCH 4/6] Geometry --- .../sql/GeometryConnectDataFrameSuite.scala | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala index 1450ac54184bd..ddc5ca7cb4fe4 100644 --- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala +++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala @@ -109,4 +109,25 @@ class GeometryConnectDataFrameSuite extends QueryTest with RemoteSparkSession { val expectedGeom = Geometry.fromWKB(point, 0) checkAnswer(df, Seq(Row(expectedGeom))) } + + test("geospatial feature disabled") { + withSQLConf("spark.sql.geospatial.enabled" -> "false") { + val geometry = Geometry.fromWKB(point1, 4326) + val schema = StructType(Seq(StructField("col1", GeometryType(4326)))) + // Java List[Row] + schema. + val javaList = java.util.Arrays.asList(Row(geometry)) + checkError( + exception = intercept[AnalysisException] { + spark.createDataFrame(javaList, schema).collect() + }, + condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED") + // Implicit encoder path. + import testImplicits._ + checkError( + exception = intercept[AnalysisException] { + Seq(geometry).toDF("g").collect() + }, + condition = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED") + } + } } From 73aef90849b74a624155d1d607d19f865882c9a8 Mon Sep 17 00:00:00 2001 From: Uros Bojanic Date: Thu, 4 Dec 2025 13:53:10 +0100 Subject: [PATCH 5/6] existsRecursively --- .../sql/connect/execution/SparkConnectPlanExecution.scala | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala index bb26e965f7151..dc20af8e3700d 100644 --- a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala +++ b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/execution/SparkConnectPlanExecution.scala @@ -128,8 +128,7 @@ private[execution] class SparkConnectPlanExecution(executeHolder: ExecuteHolder) val spark = dataframe.sparkSession val schema = dataframe.schema val geospatialEnabled = spark.sessionState.conf.geospatialEnabled - if (!geospatialEnabled && schema.fields.exists(field => - STExpressionUtils.isGeoSpatialType(field.dataType))) { + if (!geospatialEnabled && schema.existsRecursively(STExpressionUtils.isGeoSpatialType)) { throw new org.apache.spark.sql.AnalysisException( errorClass = "UNSUPPORTED_FEATURE.GEOSPATIAL_DISABLED", messageParameters = scala.collection.immutable.Map.empty) From 725274c597b6342413838822b29d033b15eb8721 Mon Sep 17 00:00:00 2001 From: Uros Bojanic Date: Thu, 4 Dec 2025 16:36:19 +0100 Subject: [PATCH 6/6] Fix test --- .../org/apache/spark/sql/GeometryConnectDataFrameSuite.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala index ddc5ca7cb4fe4..b66c8a6a3d788 100644 --- a/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala +++ b/sql/connect/client/jvm/src/test/scala/org/apache/spark/sql/GeometryConnectDataFrameSuite.scala @@ -112,8 +112,8 @@ class GeometryConnectDataFrameSuite extends QueryTest with RemoteSparkSession { test("geospatial feature disabled") { withSQLConf("spark.sql.geospatial.enabled" -> "false") { - val geometry = Geometry.fromWKB(point1, 4326) - val schema = StructType(Seq(StructField("col1", GeometryType(4326)))) + val geometry = Geometry.fromWKB(point1, 0) + val schema = StructType(Seq(StructField("col1", GeometryType(0)))) // Java List[Row] + schema. val javaList = java.util.Arrays.asList(Row(geometry)) checkError(