From eb2b0a229be86b87728a972c0a37c4dc604f899f Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Tue, 9 Jul 2024 19:30:16 +0800 Subject: [PATCH 01/10] generic udf catch exception from child udf --- .../spark/sql/hive/hiveUDFEvaluators.scala | 7 +++ .../org/apache/spark/sql/hive/hiveUDFs.scala | 16 +++--- .../sql/hive/execution/UDFCatchException.java | 51 +++++++++++++++++++ .../sql/hive/execution/UDFException.java | 26 ++++++++++ .../sql/hive/execution/HiveUDFSuite.scala | 20 ++++++++ 5 files changed, 114 insertions(+), 6 deletions(-) create mode 100644 sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java create mode 100644 sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala index 2ebe724f399a7..4f44be664658e 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala @@ -131,6 +131,13 @@ class HiveGenericUDFEvaluator( def setArg(index: Int, arg: Any): Unit = deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(arg) + def setFuncArg(index: Int, arg: () => Any): Unit = + deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(arg) + + def setException(index: Int, exp: Exception): Unit = { + deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(() => throw exp) + } + override def doEvaluate(): Any = unwrapper(function.evaluate(deferredObjects)) } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala index 6efdb676ccbdc..654e360826ee8 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala @@ -136,7 +136,7 @@ private[hive] case class HiveGenericUDF( override def eval(input: InternalRow): Any = { children.zipWithIndex.foreach { - case (child, idx) => evaluator.setArg(idx, child.eval(input)) + case (child, idx) => evaluator.setFuncArg(idx, () => child.eval(input)) } evaluator.evaluate() } @@ -157,10 +157,15 @@ private[hive] case class HiveGenericUDF( val setValues = evals.zipWithIndex.map { case (eval, i) => s""" - |if (${eval.isNull}) { - | $refEvaluator.setArg($i, null); - |} else { - | $refEvaluator.setArg($i, ${eval.value}); + |try { + | ${eval.code} + | if (${eval.isNull}) { + | $refEvaluator.setArg($i, null); + | } else { + | $refEvaluator.setArg($i, ${eval.value}); + | } + |} catch (Exception exp) { + | $refEvaluator.setException($i, exp); |} |""".stripMargin } @@ -169,7 +174,6 @@ private[hive] case class HiveGenericUDF( val resultTerm = ctx.freshName("result") ev.copy(code = code""" - |${evals.map(_.code).mkString("\n")} |${setValues.mkString("\n")} |$resultType $resultTerm = ($resultType) $refEvaluator.evaluate(); |boolean ${ev.isNull} = $resultTerm == null; diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java new file mode 100644 index 0000000000000..71455d9049978 --- /dev/null +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.execution; + +import org.apache.hadoop.hive.ql.exec.UDFArgumentException; +import org.apache.hadoop.hive.ql.udf.generic.GenericUDF; +import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; +import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory; + +public class UDFCatchException extends GenericUDF { + + @Override + public ObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { + if (args.length != 1) { + throw new UDFArgumentException("Exactly one argument is expected."); + } + return PrimitiveObjectInspectorFactory.javaStringObjectInspector; + } + + @Override + public Object evaluate(GenericUDF.DeferredObject[] args) { + if (args == null) { + return null; + } + try { + return args[0].get(); + } catch (Exception e) { + return null; + } + } + + @Override + public String getDisplayString(String[] children) { + return null; + } +} diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java new file mode 100644 index 0000000000000..211f29924ef19 --- /dev/null +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.hive.execution; + +import org.apache.hadoop.hive.ql.exec.UDF; + +public class UDFException extends UDF { + public String evaluate(String data) { + throw new RuntimeException("UDFException"); + } +} diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala index 69587a88127ee..4bbf7df254366 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala @@ -33,6 +33,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectIns import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory import org.apache.hadoop.io.{LongWritable, Writable} +import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode import org.apache.spark.{SparkException, SparkFiles, TestUtils} import org.apache.spark.sql.{AnalysisException, QueryTest, Row} import org.apache.spark.sql.catalyst.plans.logical.Project @@ -800,6 +801,25 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { } } } + + test("SPARK-48845: GenericUDF catch exceptions from child UDFs") { + withTable("test_catch_exception") { + Seq("9", "9-1").toDF("a").write.saveAsTable("test_catch_exception") + sql("create temporary function udf_exception as " + + s"'${classOf[UDFException].getName}'") + sql("create temporary function udf_catch_exception as " + + s"'${classOf[UDFCatchException].getName}'") + Seq( + CodegenObjectFactoryMode.FALLBACK.toString, + CodegenObjectFactoryMode.NO_CODEGEN.toString + ).foreach { codegenMode => + withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { + val df = sql("SELECT udf_catch_exception(udf_exception(a)) from test_catch_exception") + checkAnswer(df, Seq(Row(null), Row(null))) + } + } + } + } } class TestPair(x: Int, y: Int) extends Writable with Serializable { From 5fc7497b6390193c5903d487858c1b803ef26f84 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Tue, 9 Jul 2024 21:01:28 +0800 Subject: [PATCH 02/10] restart CI --- .../org/apache/spark/sql/hive/execution/HiveUDFSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala index 74f89d3659106..9e12265f0ce6c 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala @@ -33,9 +33,9 @@ import org.apache.hadoop.hive.serde2.objectinspector.{ObjectInspector, ObjectIns import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory import org.apache.hadoop.io.{LongWritable, Writable} -import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode import org.apache.spark.{SparkException, SparkFiles, TestUtils} import org.apache.spark.sql.{AnalysisException, QueryTest, Row} +import org.apache.spark.sql.catalyst.expressions.CodegenObjectFactoryMode import org.apache.spark.sql.catalyst.plans.logical.Project import org.apache.spark.sql.execution.WholeStageCodegenExec import org.apache.spark.sql.functions.{call_function, max} From 66f7e08b0ee92813118f7f25d8b60f28e1464028 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Wed, 10 Jul 2024 14:39:07 +0800 Subject: [PATCH 03/10] update ut for non-exception --- .../org/apache/spark/sql/hive/hiveUDFEvaluators.scala | 11 +++++------ .../scala/org/apache/spark/sql/hive/hiveUDFs.scala | 4 ++-- .../apache/spark/sql/hive/execution/UDFException.java | 6 +++++- .../spark/sql/hive/execution/HiveUDFSuite.scala | 2 +- 4 files changed, 13 insertions(+), 10 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala index 4f44be664658e..bfcaae0eb7b47 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala @@ -128,13 +128,12 @@ class HiveGenericUDFEvaluator( override def returnType: DataType = inspectorToDataType(returnInspector) - def setArg(index: Int, arg: Any): Unit = - deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(arg) + def setArg(index: Int, arg: Any): Unit = setFuncArg(index, () => arg) def setFuncArg(index: Int, arg: () => Any): Unit = deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(arg) - def setException(index: Int, exp: Exception): Unit = { + def setException(index: Int, exp: Throwable): Unit = { deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(() => throw exp) } @@ -146,10 +145,10 @@ private[hive] class DeferredObjectAdapter(oi: ObjectInspector, dataType: DataTyp extends DeferredObject with HiveInspectors { private val wrapper = wrapperFor(oi, dataType) - private var func: Any = _ - def set(func: Any): Unit = { + private var func: () => Any = _ + def set(func: () => Any): Unit = { this.func = func } override def prepare(i: Int): Unit = {} - override def get(): AnyRef = wrapper(func).asInstanceOf[AnyRef] + override def get(): AnyRef = wrapper(func()).asInstanceOf[AnyRef] } diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala index 654e360826ee8..6c8eb83ccc1dd 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala @@ -164,8 +164,8 @@ private[hive] case class HiveGenericUDF( | } else { | $refEvaluator.setArg($i, ${eval.value}); | } - |} catch (Exception exp) { - | $refEvaluator.setException($i, exp); + |} catch (Throwable t) { + | $refEvaluator.setException($i, t); |} |""".stripMargin } diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java index 211f29924ef19..3a8ba4360daf7 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java @@ -21,6 +21,10 @@ public class UDFException extends UDF { public String evaluate(String data) { - throw new RuntimeException("UDFException"); + if (Integer.parseInt(data) > 0) { + return data; + } else { + throw new RuntimeException("UDFException"); + } } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala index 9e12265f0ce6c..d3bdfa88a6cf4 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala @@ -816,7 +816,7 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { ).foreach { codegenMode => withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { val df = sql("SELECT udf_catch_exception(udf_exception(a)) from test_catch_exception") - checkAnswer(df, Seq(Row(null), Row(null))) + checkAnswer(df, Seq(Row("9"), Row(null))) } } } From f8877e5d6c1032099d6c4c78abb7eea1745273b0 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Thu, 11 Jul 2024 21:00:20 +0800 Subject: [PATCH 04/10] chagne non-codegen mode execution for GenericUDF --- .../org/apache/spark/sql/hive/hiveUDFEvaluators.scala | 6 ++---- .../main/scala/org/apache/spark/sql/hive/hiveUDFs.scala | 8 +++++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala index bfcaae0eb7b47..409be67f7af4c 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFEvaluators.scala @@ -128,10 +128,8 @@ class HiveGenericUDFEvaluator( override def returnType: DataType = inspectorToDataType(returnInspector) - def setArg(index: Int, arg: Any): Unit = setFuncArg(index, () => arg) - - def setFuncArg(index: Int, arg: () => Any): Unit = - deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(arg) + def setArg(index: Int, arg: Any): Unit = + deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(() => arg) def setException(index: Int, exp: Throwable): Unit = { deferredObjects(index).asInstanceOf[DeferredObjectAdapter].set(() => throw exp) diff --git a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala index 6c8eb83ccc1dd..227c6a618e3d4 100644 --- a/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala +++ b/sql/hive/src/main/scala/org/apache/spark/sql/hive/hiveUDFs.scala @@ -136,7 +136,13 @@ private[hive] case class HiveGenericUDF( override def eval(input: InternalRow): Any = { children.zipWithIndex.foreach { - case (child, idx) => evaluator.setFuncArg(idx, () => child.eval(input)) + case (child, idx) => + try { + evaluator.setArg(idx, child.eval(input)) + } catch { + case t: Throwable => + evaluator.setException(idx, t) + } } evaluator.evaluate() } From b7ed13348ac8f470b1794ef1b97eb795cf186ab0 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Thu, 11 Jul 2024 21:23:35 +0800 Subject: [PATCH 05/10] add withUserDefinedFunction in UT --- .../sql/hive/execution/HiveUDFSuite.scala | 26 ++++++++++--------- 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala index d3bdfa88a6cf4..6bbe634e17a28 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala @@ -805,18 +805,20 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { test("SPARK-48845: GenericUDF catch exceptions from child UDFs") { withTable("test_catch_exception") { - Seq("9", "9-1").toDF("a").write.saveAsTable("test_catch_exception") - sql("create temporary function udf_exception as " + - s"'${classOf[UDFException].getName}'") - sql("create temporary function udf_catch_exception as " + - s"'${classOf[UDFCatchException].getName}'") - Seq( - CodegenObjectFactoryMode.FALLBACK.toString, - CodegenObjectFactoryMode.NO_CODEGEN.toString - ).foreach { codegenMode => - withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { - val df = sql("SELECT udf_catch_exception(udf_exception(a)) from test_catch_exception") - checkAnswer(df, Seq(Row("9"), Row(null))) + withUserDefinedFunction("udf_exception" -> true, "udf_catch_exception" -> true) { + Seq("9", "9-1").toDF("a").write.saveAsTable("test_catch_exception") + sql("CREATE TEMPORARY FUNCTION udf_exception AS " + + s"'${classOf[UDFException].getName}'") + sql("CREATE TEMPORARY FUNCTION udf_catch_exception AS " + + s"'${classOf[UDFCatchException].getName}'") + Seq( + CodegenObjectFactoryMode.FALLBACK.toString, + CodegenObjectFactoryMode.NO_CODEGEN.toString + ).foreach { codegenMode => + withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { + val df = sql("SELECT udf_catch_exception(udf_exception(a)) FROM test_catch_exception") + checkAnswer(df, Seq(Row("9"), Row(null))) + } } } } From 0465984fc080b5acbdf37eacbd6bd044f6777239 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Fri, 12 Jul 2024 09:22:08 +0800 Subject: [PATCH 06/10] reformat code --- .../sql/hive/execution/UDFCatchException.java | 42 +++++++++---------- ...FException.java => UDFThrowException.java} | 16 +++---- .../sql/hive/execution/HiveUDFSuite.scala | 9 ++-- 3 files changed, 34 insertions(+), 33 deletions(-) rename sql/hive/src/test/java/org/apache/spark/sql/hive/execution/{UDFException.java => UDFThrowException.java} (75%) diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java index 71455d9049978..0c076cd880e75 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -24,28 +24,28 @@ public class UDFCatchException extends GenericUDF { - @Override - public ObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { - if (args.length != 1) { - throw new UDFArgumentException("Exactly one argument is expected."); - } - return PrimitiveObjectInspectorFactory.javaStringObjectInspector; + @Override + public ObjectInspector initialize(ObjectInspector[] args) throws UDFArgumentException { + if (args.length != 1) { + throw new UDFArgumentException("Exactly one argument is expected."); } + return PrimitiveObjectInspectorFactory.javaStringObjectInspector; + } - @Override - public Object evaluate(GenericUDF.DeferredObject[] args) { - if (args == null) { - return null; - } - try { - return args[0].get(); - } catch (Exception e) { - return null; - } + @Override + public Object evaluate(GenericUDF.DeferredObject[] args) { + if (args == null) { + return null; } - - @Override - public String getDisplayString(String[] children) { - return null; + try { + return args[0].get(); + } catch (Exception e) { + return null; } + } + + @Override + public String getDisplayString(String[] children) { + return null; + } } diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java similarity index 75% rename from sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java rename to sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java index 3a8ba4360daf7..59f7d9020b55e 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -19,12 +19,12 @@ import org.apache.hadoop.hive.ql.exec.UDF; -public class UDFException extends UDF { - public String evaluate(String data) { - if (Integer.parseInt(data) > 0) { - return data; - } else { - throw new RuntimeException("UDFException"); - } +public class UDFThrowException extends UDF { + public String evaluate(String data) { + if (Integer.parseInt(data) > 0) { + return data; + } else { + throw new RuntimeException("UDFException"); } + } } diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala index 6bbe634e17a28..2e88b13f0963d 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveUDFSuite.scala @@ -805,10 +805,10 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { test("SPARK-48845: GenericUDF catch exceptions from child UDFs") { withTable("test_catch_exception") { - withUserDefinedFunction("udf_exception" -> true, "udf_catch_exception" -> true) { + withUserDefinedFunction("udf_throw_exception" -> true, "udf_catch_exception" -> true) { Seq("9", "9-1").toDF("a").write.saveAsTable("test_catch_exception") - sql("CREATE TEMPORARY FUNCTION udf_exception AS " + - s"'${classOf[UDFException].getName}'") + sql("CREATE TEMPORARY FUNCTION udf_throw_exception AS " + + s"'${classOf[UDFThrowException].getName}'") sql("CREATE TEMPORARY FUNCTION udf_catch_exception AS " + s"'${classOf[UDFCatchException].getName}'") Seq( @@ -816,7 +816,8 @@ class HiveUDFSuite extends QueryTest with TestHiveSingleton with SQLTestUtils { CodegenObjectFactoryMode.NO_CODEGEN.toString ).foreach { codegenMode => withSQLConf(SQLConf.CODEGEN_FACTORY_MODE.key -> codegenMode) { - val df = sql("SELECT udf_catch_exception(udf_exception(a)) FROM test_catch_exception") + val df = sql( + "SELECT udf_catch_exception(udf_throw_exception(a)) FROM test_catch_exception") checkAnswer(df, Seq(Row("9"), Row(null))) } } From d3ef30261f40d82e5e6bd6e52ab7fec2dbeb9285 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Fri, 12 Jul 2024 10:06:34 +0800 Subject: [PATCH 07/10] reformat license code --- .../org/apache/spark/sql/hive/execution/UDFCatchException.java | 2 +- .../org/apache/spark/sql/hive/execution/UDFThrowException.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java index 0c076cd880e75..242dbeaa63c94 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFCatchException.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java index 59f7d9020b55e..85b66dbc8e3b7 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java @@ -6,7 +6,7 @@ * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * - * http://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, From 0203bcca0e50cefc2e7d643b43dcdcf37407cc5b Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 12 Jul 2024 11:30:07 +0800 Subject: [PATCH 08/10] Update sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java --- .../apache/spark/sql/hive/execution/UDFThrowException.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java index 85b66dbc8e3b7..162411bb4708a 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java @@ -21,10 +21,6 @@ public class UDFThrowException extends UDF { public String evaluate(String data) { - if (Integer.parseInt(data) > 0) { - return data; - } else { - throw new RuntimeException("UDFException"); - } + Integer.parseInt(data) } } From 6c47f98d8f96ddfea6dbb642e91d2fa0fb96e047 Mon Sep 17 00:00:00 2001 From: Kent Yao Date: Fri, 12 Jul 2024 11:31:07 +0800 Subject: [PATCH 09/10] Update sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java --- .../org/apache/spark/sql/hive/execution/UDFThrowException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java index 162411bb4708a..16ce2942bc55e 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java @@ -21,6 +21,6 @@ public class UDFThrowException extends UDF { public String evaluate(String data) { - Integer.parseInt(data) + return Integer.parseInt(data); } } From c15482340b1787e07b7524c54bb2914038262939 Mon Sep 17 00:00:00 2001 From: jackylee-ch Date: Fri, 12 Jul 2024 16:36:41 +0800 Subject: [PATCH 10/10] fix code --- .../org/apache/spark/sql/hive/execution/UDFThrowException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java index 16ce2942bc55e..5d6ff6ca40ae5 100644 --- a/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java +++ b/sql/hive/src/test/java/org/apache/spark/sql/hive/execution/UDFThrowException.java @@ -21,6 +21,6 @@ public class UDFThrowException extends UDF { public String evaluate(String data) { - return Integer.parseInt(data); + return Integer.valueOf(data).toString(); } }