From b72664a8eb2028b653dd8ead60531b8fed17fd50 Mon Sep 17 00:00:00 2001 From: Mihailo Milosevic Date: Thu, 11 Apr 2024 21:36:00 +0200 Subject: [PATCH 1/4] Fix ANSI tests --- .../apache/spark/sql/catalyst/analysis/TypeCoercion.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 5d8bcd4f7b1f9..0ea0a35ed8e8d 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -18,7 +18,6 @@ package org.apache.spark.sql.catalyst.analysis import javax.annotation.Nullable - import scala.annotation.tailrec import scala.collection.mutable @@ -31,7 +30,7 @@ import org.apache.spark.sql.catalyst.trees.AlwaysProcess import org.apache.spark.sql.catalyst.types.DataTypeUtils import org.apache.spark.sql.errors.QueryCompilationErrors import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.internal.types.{AbstractArrayType, AbstractStringType} +import org.apache.spark.sql.internal.types.{AbstractArrayType, AbstractStringType, StringTypeAnyCollation} import org.apache.spark.sql.types._ import org.apache.spark.sql.types.UpCastRule.numericPrecedence @@ -422,7 +421,8 @@ abstract class TypeCoercionBase { case None => c } - case aj @ ArrayJoin(arr, d, nr) if !ArrayType(StringType).acceptsType(arr.dataType) && + case aj @ ArrayJoin(arr, d, nr) + if !AbstractArrayType(StringTypeAnyCollation).acceptsType(arr.dataType) && ArrayType.acceptsType(arr.dataType) => val containsNull = arr.dataType.asInstanceOf[ArrayType].containsNull implicitCast(arr, ArrayType(StringType, containsNull)) match { From 6047ac98b1c640cb1d3c0961456a47c59a9f5ade Mon Sep 17 00:00:00 2001 From: Mihailo Milosevic Date: Thu, 11 Apr 2024 21:39:47 +0200 Subject: [PATCH 2/4] Fix imports --- .../org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 0ea0a35ed8e8d..4349ba7a24f01 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -18,6 +18,7 @@ package org.apache.spark.sql.catalyst.analysis import javax.annotation.Nullable + import scala.annotation.tailrec import scala.collection.mutable From 9cd833285f9a8157cecfe1e874ee51089ec18ba0 Mon Sep 17 00:00:00 2001 From: Mihailo Milosevic Date: Thu, 11 Apr 2024 23:18:43 +0200 Subject: [PATCH 3/4] Fix implicitCast in ANSITypeCoercion --- .../spark/sql/catalyst/analysis/AnsiTypeCoercion.scala | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala index 52c1136b1ee30..9989ca79ed270 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/AnsiTypeCoercion.scala @@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.analysis import org.apache.spark.sql.catalyst.expressions._ import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan import org.apache.spark.sql.catalyst.rules.Rule -import org.apache.spark.sql.internal.types.AbstractStringType +import org.apache.spark.sql.internal.types.{AbstractArrayType, AbstractStringType} import org.apache.spark.sql.types._ import org.apache.spark.sql.types.UpCastRule.numericPrecedence @@ -197,6 +197,9 @@ object AnsiTypeCoercion extends TypeCoercionBase { case (_: StringType, a @ (_: AtomicType | NumericType | DecimalType | AnyTimestampType)) => Some(a.defaultConcreteType) + case (ArrayType(fromType, _), AbstractArrayType(toType)) => + Some(implicitCast(fromType, toType).map(ArrayType(_, true)).orNull) + // When the target type is `TypeCollection`, there is another branch to find the // "closet convertible data type" below. case (_, target) if !target.isInstanceOf[TypeCollection] => From 106e0d83e0b75bae43dce03030d1d5ccf853f593 Mon Sep 17 00:00:00 2001 From: Dongjoon Hyun Date: Thu, 11 Apr 2024 15:09:05 -0700 Subject: [PATCH 4/4] Indentation --- .../org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala index 4349ba7a24f01..259e28b62bca7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/TypeCoercion.scala @@ -423,8 +423,8 @@ abstract class TypeCoercionBase { } case aj @ ArrayJoin(arr, d, nr) - if !AbstractArrayType(StringTypeAnyCollation).acceptsType(arr.dataType) && - ArrayType.acceptsType(arr.dataType) => + if !AbstractArrayType(StringTypeAnyCollation).acceptsType(arr.dataType) && + ArrayType.acceptsType(arr.dataType) => val containsNull = arr.dataType.asInstanceOf[ArrayType].containsNull implicitCast(arr, ArrayType(StringType, containsNull)) match { case Some(castedArr) => ArrayJoin(castedArr, d, nr)