diff --git a/docs/sql-migration-guide.md b/docs/sql-migration-guide.md index 42df05f7f7068..ecd850e11a8ac 100644 --- a/docs/sql-migration-guide.md +++ b/docs/sql-migration-guide.md @@ -27,6 +27,7 @@ license: | - Since Spark 3.4, Number or Number(\*) from Teradata will be treated as Decimal(38,18). In Spark 3.3 or earlier, Number or Number(\*) from Teradata will be treated as Decimal(38, 0), in which case the fractional part will be removed. - Since Spark 3.4, v1 database, table, permanent view and function identifier will include 'spark_catalog' as the catalog name if database is defined, e.g. a table identifier will be: `spark_catalog.default.t`. To restore the legacy behavior, set `spark.sql.legacy.v1IdentifierNoCatalog` to `true`. - Since Spark 3.4, when ANSI SQL mode(configuration `spark.sql.ansi.enabled`) is on, Spark SQL always returns NULL result on getting a map value with a non-existing key. In Spark 3.3 or earlier, there will be an error. + - Since Spark 3.4, the SQL CLI `spark-sql` does not print the prefix `Error in query:` before the error message of `AnalysisException`. ## Upgrading from Spark SQL 3.2 to 3.3 diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 0839a2f487511..31bdbca4a256e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -3879,7 +3879,7 @@ object SQLConf { .doc("When PRETTY, the error message consists of textual representation of error class, " + "message and query context. The MINIMAL and STANDARD formats are pretty JSON formats where " + "STANDARD includes an additional JSON field `message`. This configuration property " + - "influences on error messages of Thrift Server while running queries.") + "influences on error messages of Thrift Server and SQL CLI while running queries.") .version("3.4.0") .stringConf.transform(_.toUpperCase(Locale.ROOT)) .checkValues(ErrorMessageFormat.values.map(_.toString)) diff --git a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala index d40cf73be6354..fcd5c56e0fac6 100644 --- a/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala +++ b/sql/hive-thriftserver/src/main/scala/org/apache/spark/sql/hive/thriftserver/SparkSQLCLIDriver.scala @@ -39,7 +39,7 @@ import org.apache.thrift.transport.TSocket import org.slf4j.LoggerFactory import sun.misc.{Signal, SignalHandler} -import org.apache.spark.SparkConf +import org.apache.spark.{ErrorMessageFormat, SparkConf, SparkThrowable, SparkThrowableHelper} import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.internal.Logging import org.apache.spark.sql.AnalysisException @@ -394,19 +394,17 @@ private[hive] class SparkSQLCLIDriver extends CliDriver with Logging { ret = rc.getResponseCode if (ret != 0) { - rc.getException match { - case e: AnalysisException => e.cause match { - case Some(_) if !sessionState.getIsSilent => - err.println( - s"""Error in query: ${e.getMessage} - |${org.apache.hadoop.util.StringUtils.stringifyException(e)} - """.stripMargin) - // For analysis exceptions in silent mode or simple ones that only related to the - // query itself, such as `NoSuchDatabaseException`, only the error is printed out - // to the console. - case _ => err.println(s"""Error in query: ${e.getMessage}""") - } - case _ => err.println(rc.getErrorMessage()) + val format = SparkSQLEnv.sqlContext.conf.errorMessageFormat + val e = rc.getException + val msg = e match { + case st: SparkThrowable with Throwable => SparkThrowableHelper.getMessage(st, format) + case _ => e.getMessage + } + err.println(msg) + if (format == ErrorMessageFormat.PRETTY && + !sessionState.getIsSilent && + (!e.isInstanceOf[AnalysisException] || e.getCause != null)) { + e.printStackTrace(err) } driver.close() return ret diff --git a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala index e64492c3128fa..6bbc26bc8caab 100644 --- a/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala +++ b/sql/hive-thriftserver/src/test/scala/org/apache/spark/sql/hive/thriftserver/CliSuite.scala @@ -32,14 +32,14 @@ import org.apache.hadoop.hive.cli.CliSessionState import org.apache.hadoop.hive.conf.HiveConf.ConfVars import org.apache.hadoop.hive.ql.session.SessionState -import org.apache.spark.{SparkConf, SparkContext, SparkFunSuite} +import org.apache.spark.{ErrorMessageFormat, SparkConf, SparkContext, SparkFunSuite} import org.apache.spark.ProcessTestUtils.ProcessOutputCapturer import org.apache.spark.deploy.SparkHadoopUtil import org.apache.spark.sql.hive.HiveUtils import org.apache.spark.sql.hive.HiveUtils._ import org.apache.spark.sql.hive.client.HiveClientImpl import org.apache.spark.sql.hive.test.HiveTestJars -import org.apache.spark.sql.internal.StaticSQLConf +import org.apache.spark.sql.internal.{SQLConf, StaticSQLConf} import org.apache.spark.util.{ThreadUtils, Utils} /** @@ -388,8 +388,7 @@ class CliSuite extends SparkFunSuite { test("SPARK-11188 Analysis error reporting") { runCliWithin(timeout = 2.minute, errorResponses = Seq("AnalysisException"))( - "select * from nonexistent_table;" - -> "Error in query: Table or view not found: nonexistent_table;" + "select * from nonexistent_table;" -> "Table or view not found: nonexistent_table;" ) } @@ -698,4 +697,79 @@ class CliSuite extends SparkFunSuite { t.start() cd.await() } + + // scalastyle:off line.size.limit + test("formats of error messages") { + def check(format: ErrorMessageFormat.Value, errorMessage: String, silent: Boolean): Unit = { + val expected = errorMessage.split(System.lineSeparator()).map("" -> _) + runCliWithin( + 1.minute, + extraArgs = Seq( + "--conf", s"spark.hive.session.silent=$silent", + "--conf", s"${SQLConf.ERROR_MESSAGE_FORMAT.key}=$format", + "--conf", s"${SQLConf.ANSI_ENABLED.key}=true", + "-e", "select 1 / 0"), + errorResponses = Seq("DIVIDE_BY_ZERO"))(expected: _*) + } + check( + format = ErrorMessageFormat.PRETTY, + errorMessage = + """[DIVIDE_BY_ZERO] Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. + |== SQL(line 1, position 8) == + |select 1 / 0 + | ^^^^^ + |""".stripMargin, + silent = true) + check( + format = ErrorMessageFormat.PRETTY, + errorMessage = + """[DIVIDE_BY_ZERO] Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. + |== SQL(line 1, position 8) == + |select 1 / 0 + | ^^^^^ + | + |org.apache.spark.SparkArithmeticException: [DIVIDE_BY_ZERO] Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead. If necessary set "spark.sql.ansi.enabled" to "false" to bypass this error. + |""".stripMargin, + silent = false) + Seq(true, false).foreach { silent => + check( + format = ErrorMessageFormat.MINIMAL, + errorMessage = + """{ + | "errorClass" : "DIVIDE_BY_ZERO", + | "sqlState" : "22012", + | "messageParameters" : { + | "config" : "\"spark.sql.ansi.enabled\"" + | }, + | "queryContext" : [ { + | "objectType" : "", + | "objectName" : "", + | "startIndex" : 8, + | "stopIndex" : 12, + | "fragment" : "1 / 0" + | } ] + |}""".stripMargin, + silent) + check( + format = ErrorMessageFormat.STANDARD, + errorMessage = + """{ + | "errorClass" : "DIVIDE_BY_ZERO", + | "message" : "Division by zero. Use `try_divide` to tolerate divisor being 0 and return NULL instead. If necessary set to \"false\" to bypass this error.", + | "sqlState" : "22012", + | "messageParameters" : { + | "config" : "\"spark.sql.ansi.enabled\"" + | }, + | "queryContext" : [ { + | "objectType" : "", + | "objectName" : "", + | "startIndex" : 8, + | "stopIndex" : 12, + | "fragment" : "1 / 0" + | } ] + |}""".stripMargin, + silent) + } + } + // scalastyle:on line.size.limit }