diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala index 1c599af2a01d..d629eb701712 100755 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringExpressions.scala @@ -1372,10 +1372,10 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC val pattern = children.head.genCode(ctx) val argListGen = children.tail.map(x => (x.dataType, x.genCode(ctx))) - val argListCode = argListGen.map(_._2.code + "\n") - - val argListString = argListGen.foldLeft("")((s, v) => { - val nullSafeString = + val argList = ctx.freshName("argLists") + val numArgLists = argListGen.length + val argListCode = argListGen.zipWithIndex.map { case(v, index) => + val value = if (ctx.boxedType(v._1) != ctx.javaType(v._1)) { // Java primitives get boxed in order to allow null values. s"(${v._2.isNull}) ? (${ctx.boxedType(v._1)}) null : " + @@ -1383,8 +1383,19 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC } else { s"(${v._2.isNull}) ? null : ${v._2.value}" } - s + "," + nullSafeString - }) + s""" + ${v._2.code} + $argList[$index] = $value; + """ + } + val argListCodes = if (ctx.INPUT_ROW != null && ctx.currentVars == null) { + ctx.splitExpressions( + expressions = argListCode, + funcName = "valueFormatString", + arguments = ("InternalRow", ctx.INPUT_ROW) :: ("Object[]", argList) :: Nil) + } else { + argListCode.mkString("\n") + } val form = ctx.freshName("formatter") val formatter = classOf[java.util.Formatter].getName @@ -1395,10 +1406,11 @@ case class FormatString(children: Expression*) extends Expression with ImplicitC boolean ${ev.isNull} = ${pattern.isNull}; ${ctx.javaType(dataType)} ${ev.value} = ${ctx.defaultValue(dataType)}; if (!${ev.isNull}) { - ${argListCode.mkString} $stringBuffer $sb = new $stringBuffer(); $formatter $form = new $formatter($sb, ${classOf[Locale].getName}.US); - $form.format(${pattern.value}.toString() $argListString); + Object[] $argList = new Object[$numArgLists]; + $argListCodes + $form.format(${pattern.value}.toString(), $argList); ${ev.value} = UTF8String.fromString($sb.toString()); }""") } diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala index c76139475687..54cde77176e2 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/StringExpressionsSuite.scala @@ -518,6 +518,14 @@ class StringExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper { FormatString(Literal("aa%d%s"), 12, Literal.create(null, StringType)), "aa12null") } + test("SPARK-22603: FormatString should not generate codes beyond 64KB") { + val N = 4500 + val args = (1 to N).map(i => Literal.create(i.toString, StringType)) + val format = "%s" * N + val expected = (1 to N).map(i => i.toString).mkString + checkEvaluation(FormatString(Literal(format) +: args: _*), expected) + } + test("INSTR") { val s1 = 'a.string.at(0) val s2 = 'b.string.at(1)