From 86e60397ac5958cd78e4c7fe3c54684623bbfcb3 Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Thu, 25 Apr 2019 11:26:09 +0800 Subject: [PATCH 1/2] automatically get the latest Spark versions in HiveExternalCatalogVersionsSuite --- .../sql/hive/HiveExternalCatalogVersionsSuite.scala | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 0a05ec5146ffa..004dcb19c57ce 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -206,7 +206,16 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { object PROCESS_TABLES extends QueryTest with SQLTestUtils { // Tests the latest version of every release line. - val testingVersions = Seq("2.3.3", "2.4.2") + lazy val testingVersions: Seq[String] = { + import scala.io.Source + val versions = Source.fromURL("https://dist.apache.org/repos/dist/release/spark/").mkString + .split("\n") + .filter(_.contains("""
  • """.r.findFirstMatchIn(_).get.group(1)) + .filter(_ < org.apache.spark.SPARK_VERSION) + logInfo(s"Testing ${org.apache.spark.SPARK_VERSION} with ${versions.mkString(", ")}.") + versions + } protected var spark: SparkSession = _ From 8700851bd16146014ca0faaa1006d55c80d28c9e Mon Sep 17 00:00:00 2001 From: Wenchen Fan Date: Thu, 25 Apr 2019 15:37:13 +0800 Subject: [PATCH 2/2] address comments --- .../HiveExternalCatalogVersionsSuite.scala | 24 ++++++++++++------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala index 004dcb19c57ce..ec10295358d55 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveExternalCatalogVersionsSuite.scala @@ -22,6 +22,7 @@ import java.nio.charset.StandardCharsets import java.nio.file.{Files, Paths} import scala.sys.process._ +import scala.util.control.NonFatal import org.apache.hadoop.conf.Configuration @@ -169,6 +170,10 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { """.stripMargin.getBytes("utf8")) // scalastyle:on line.size.limit + if (PROCESS_TABLES.testingVersions.isEmpty) { + fail("Fail to get the lates Spark versions to test.") + } + PROCESS_TABLES.testingVersions.zipWithIndex.foreach { case (version, index) => val sparkHome = new File(sparkTestingDir, s"spark-$version") if (!sparkHome.exists()) { @@ -206,15 +211,18 @@ class HiveExternalCatalogVersionsSuite extends SparkSubmitTestUtils { object PROCESS_TABLES extends QueryTest with SQLTestUtils { // Tests the latest version of every release line. - lazy val testingVersions: Seq[String] = { + val testingVersions: Seq[String] = { import scala.io.Source - val versions = Source.fromURL("https://dist.apache.org/repos/dist/release/spark/").mkString - .split("\n") - .filter(_.contains("""
  • """.r.findFirstMatchIn(_).get.group(1)) - .filter(_ < org.apache.spark.SPARK_VERSION) - logInfo(s"Testing ${org.apache.spark.SPARK_VERSION} with ${versions.mkString(", ")}.") - versions + try { + Source.fromURL("https://dist.apache.org/repos/dist/release/spark/").mkString + .split("\n") + .filter(_.contains("""
  • """.r.findFirstMatchIn(_).get.group(1)) + .filter(_ < org.apache.spark.SPARK_VERSION) + } catch { + // do not throw exception during object initialization. + case NonFatal(_) => Nil + } } protected var spark: SparkSession = _