diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index cdeb27d0c289..3259b59b34c1 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -547,14 +547,6 @@ case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: E override def dataType: DataType = MapType(StringType, StringType) - override def checkInputDataTypes(): TypeCheckResult = { - if (Seq(pairDelim, keyValueDelim).exists(! _.foldable)) { - TypeCheckResult.TypeCheckFailure(s"$prettyName's delimiters must be foldable.") - } else { - super.checkInputDataTypes() - } - } - private lazy val mapBuilder = new ArrayBasedMapBuilder(StringType, StringType) override def nullSafeEval( diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala index 7b482dec7e22..e3afd6a3bb3c 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala @@ -476,6 +476,10 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { val m5 = Map("a" -> null) checkEvaluation(new StringToMap(s5), m5) + val s6 = Literal("a=1&b=2&c=3") + val m6 = Map("a" -> "1", "b" -> "2", "c" -> "3") + checkEvaluation(new StringToMap(s6, NonFoldableLiteral("&"), NonFoldableLiteral("=")), m6) + checkExceptionInExpression[RuntimeException]( new StringToMap(Literal("a:1,b:2,a:3")), "Duplicate map key") withSQLConf(SQLConf.MAP_KEY_DEDUP_POLICY.key -> SQLConf.MapKeyDedupPolicy.LAST_WIN.toString) { @@ -492,12 +496,6 @@ class ComplexTypeSuite extends SparkFunSuite with ExpressionEvalHelper { assert(StringToMap(Literal("a:1,b:2,c:3"), Literal(null), Literal(null)) .checkInputDataTypes().isFailure) assert(new StringToMap(Literal(null), Literal(null)).checkInputDataTypes().isFailure) - - assert(new StringToMap(Literal("a:1_b:2_c:3"), NonFoldableLiteral("_")) - .checkInputDataTypes().isFailure) - assert( - new StringToMap(Literal("a=1_b=2_c=3"), Literal("_"), NonFoldableLiteral("=")) - .checkInputDataTypes().isFailure) } test("SPARK-22693: CreateNamedStruct should not use global variables") { diff --git a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala index d07be9c19714..082358b744ac 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/StringFunctionsSuite.scala @@ -606,6 +606,36 @@ class StringFunctionsSuite extends QueryTest with SharedSparkSession { Seq(Row(Map("a" -> "1", "b" -> "2", "c" -> "3"))) ) + checkAnswer( + df2.selectExpr("str_to_map(a, ',')"), + Seq(Row(Map("a" -> "1", "b" -> "2", "c" -> "3"))) + ) + + val df3 = Seq( + ("a=1&b=2", "&", "="), + ("k#2%v#3", "%", "#") + ).toDF("str", "delim1", "delim2") + + checkAnswer( + df3.selectExpr("str_to_map(str, delim1, delim2)"), + Seq( + Row(Map("a" -> "1", "b" -> "2")), + Row(Map("k" -> "2", "v" -> "3")) + ) + ) + + val df4 = Seq( + ("a:1&b:2", "&"), + ("k:2%v:3", "%") + ).toDF("str", "delim1") + + checkAnswer( + df4.selectExpr("str_to_map(str, delim1)"), + Seq( + Row(Map("a" -> "1", "b" -> "2")), + Row(Map("k" -> "2", "v" -> "3")) + ) + ) } test("SPARK-36148: check input data types of regexp_replace") {