From 110bdd8e4abd89056b9811ed9d7c0f43e5279d70 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Sat, 23 May 2020 16:35:18 +0900 Subject: [PATCH 1/4] Added RealBrowserUIHistoryServerSuite. --- .../deploy/history/HistoryServerSuite.scala | 62 ------- .../RealBrowserUIHistoryServerSuite.scala | 158 ++++++++++++++++++ 2 files changed, 158 insertions(+), 62 deletions(-) create mode 100644 core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index 8c2dfff4112c0..8737cd5bb3241 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -29,8 +29,6 @@ import scala.concurrent.duration._ import com.google.common.io.{ByteStreams, Files} import org.apache.commons.io.{FileUtils, IOUtils} import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} -import org.eclipse.jetty.proxy.ProxyServlet -import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.json4s.JsonAST._ import org.json4s.jackson.JsonMethods import org.json4s.jackson.JsonMethods._ @@ -336,66 +334,6 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers assert(response.contains(SPARK_VERSION)) } - test("ajax rendered relative links are prefixed with uiRoot (spark.ui.proxyBase)") { - val uiRoot = "/testwebproxybase" - System.setProperty("spark.ui.proxyBase", uiRoot) - - stop() - init() - - val port = server.boundPort - - val servlet = new ProxyServlet { - override def rewriteTarget(request: HttpServletRequest): String = { - // servlet acts like a proxy that redirects calls made on - // spark.ui.proxyBase context path to the normal servlet handlers operating off "/" - val sb = request.getRequestURL() - - if (request.getQueryString() != null) { - sb.append(s"?${request.getQueryString()}") - } - - val proxyidx = sb.indexOf(uiRoot) - sb.delete(proxyidx, proxyidx + uiRoot.length).toString - } - } - - val contextHandler = new ServletContextHandler - val holder = new ServletHolder(servlet) - contextHandler.setContextPath(uiRoot) - contextHandler.addServlet(holder, "/") - server.attachHandler(contextHandler) - - implicit val webDriver: WebDriver = new HtmlUnitDriver(true) - - try { - val url = s"http://localhost:$port" - - go to s"$url$uiRoot" - - // expect the ajax call to finish in 5 seconds - implicitlyWait(org.scalatest.time.Span(5, org.scalatest.time.Seconds)) - - // once this findAll call returns, we know the ajax load of the table completed - findAll(ClassNameQuery("odd")) - - val links = findAll(TagNameQuery("a")) - .map(_.attribute("href")) - .filter(_.isDefined) - .map(_.get) - .filter(_.startsWith(url)).toList - - // there are at least some URL links that were generated via javascript, - // and they all contain the spark.ui.proxyBase (uiRoot) - links.length should be > 4 - all(links) should startWith(url + uiRoot) - } finally { - contextHandler.stop() - quit() - } - - } - /** * Verify that the security manager needed for the history server can be instantiated * when `spark.authenticate` is `true`, rather than raise an `IllegalArgumentException`. diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala new file mode 100644 index 0000000000000..a24c7b8d8ae55 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.history + +import javax.servlet.http.HttpServletRequest + +import org.eclipse.jetty.proxy.ProxyServlet +import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} +import org.openqa.selenium.WebDriver +import org.openqa.selenium.htmlunit.HtmlUnitDriver +import org.scalatest._ +import org.scalatestplus.selenium.WebBrowser + +import org.apache.spark._ +import org.apache.spark.internal.config.{EVENT_LOG_STAGE_EXECUTOR_METRICS, EXECUTOR_PROCESS_TREE_METRICS_ENABLED} +import org.apache.spark.internal.config.History.{HISTORY_LOG_DIR, LOCAL_STORE_DIR, UPDATE_INTERVAL_S} +import org.apache.spark.internal.config.Tests.IS_TESTING +import org.apache.spark.util.{ResetSystemProperties, Utils} + +/** + * Tests for HistoryServer with read web browsers. + */ +abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) + extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll + with BeforeAndAfterEach with ResetSystemProperties { + + implicit var webDriver: WebDriver + + private val driverPropPrefix = "spark.test." + private val logDir = getTestResourcePath("spark-events") + private val storeDir = Utils.createTempDir(namePrefix = "history") + + private var provider: FsHistoryProvider = null + private var server: HistoryServer = null + private var port: Int = -1 + + override def beforeAll() { + super.beforeAll() + assume( + sys.props(driverPropPrefix + driverProp) !== null, + "System property " + driverPropPrefix + driverProp + + " should be set to the corresponding driver path.") + sys.props(driverProp) = sys.props(driverPropPrefix + driverProp) + } + + override def beforeEach { + super.beforeEach() + if (server == null) { + init() + } + } + + override def afterAll(): Unit = { + sys.props.remove(driverProp) + super.afterAll() + } + + def init(extraConf: (String, String)*): Unit = { + Utils.deleteRecursively(storeDir) + assert(storeDir.mkdir()) + val conf = new SparkConf() + .set(HISTORY_LOG_DIR, logDir) + .set(UPDATE_INTERVAL_S.key, "0") + .set(IS_TESTING, true) + .set(LOCAL_STORE_DIR, storeDir.getAbsolutePath()) + .set(EVENT_LOG_STAGE_EXECUTOR_METRICS, true) + .set(EXECUTOR_PROCESS_TREE_METRICS_ENABLED, true) + conf.setAll(extraConf) + provider = new FsHistoryProvider(conf) + provider.checkForLogs() + val securityManager = HistoryServer.createSecurityManager(conf) + + server = new HistoryServer(conf, provider, securityManager, 18080) + server.initialize() + server.bind() + provider.start() + port = server.boundPort + } + + def stop(): Unit = { + server.stop() + server = null + } + + test("ajax rendered relative links are prefixed with uiRoot (spark.ui.proxyBase)") { + val uiRoot = "/testwebproxybase" + System.setProperty("spark.ui.proxyBase", uiRoot) + + stop() + init() + + val port = server.boundPort + + val servlet = new ProxyServlet { + override def rewriteTarget(request: HttpServletRequest): String = { + // servlet acts like a proxy that redirects calls made on + // spark.ui.proxyBase context path to the normal servlet handlers operating off "/" + val sb = request.getRequestURL() + + if (request.getQueryString() != null) { + sb.append(s"?${request.getQueryString()}") + } + + val proxyidx = sb.indexOf(uiRoot) + sb.delete(proxyidx, proxyidx + uiRoot.length).toString + } + } + + val contextHandler = new ServletContextHandler + val holder = new ServletHolder(servlet) + contextHandler.setContextPath(uiRoot) + contextHandler.addServlet(holder, "/") + server.attachHandler(contextHandler) + + implicit val webDriver: WebDriver = new HtmlUnitDriver(true) + + try { + val url = s"http://localhost:$port" + + go to s"$url$uiRoot" + + // expect the ajax call to finish in 5 seconds + implicitlyWait(org.scalatest.time.Span(5, org.scalatest.time.Seconds)) + + // once this findAll call returns, we know the ajax load of the table completed + findAll(ClassNameQuery("odd")) + + val links = findAll(TagNameQuery("a")) + .map(_.attribute("href")) + .filter(_.isDefined) + .map(_.get) + .filter(_.startsWith(url)).toList + + // there are at least some URL links that were generated via javascript, + // and they all contain the spark.ui.proxyBase (uiRoot) + links.length should be > 4 + all(links) should startWith(url + uiRoot) + } finally { + contextHandler.stop() + quit() + } + } +} From e5ee94edfd037b34168dd37ab61e1fc7cbd7fa90 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Sat, 23 May 2020 16:35:48 +0900 Subject: [PATCH 2/4] Added ChromeUIHistoryServerSuite. --- .../history/ChromeUIHistoryServerSuite.scala | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala new file mode 100644 index 0000000000000..1fa2d0ab882c9 --- /dev/null +++ b/core/src/test/scala/org/apache/spark/deploy/history/ChromeUIHistoryServerSuite.scala @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.deploy.history + +import org.openqa.selenium.WebDriver +import org.openqa.selenium.chrome.{ChromeDriver, ChromeOptions} + +import org.apache.spark.tags.ChromeUITest + +/** + * Tests for HistoryServer with Chrome. + */ +@ChromeUITest +class ChromeUIHistoryServerSuite + extends RealBrowserUIHistoryServerSuite("webdriver.chrome.driver") { + + override var webDriver: WebDriver = _ + + override def beforeAll(): Unit = { + super.beforeAll() + val chromeOptions = new ChromeOptions + chromeOptions.addArguments("--headless", "--disable-gpu") + webDriver = new ChromeDriver(chromeOptions) + } + + override def afterAll(): Unit = { + try { + if (webDriver != null) { + webDriver.quit() + } + } finally { + super.afterAll() + } + } +} From 63d530aeec1eda01c038554b2c4d66d7358de605 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Sat, 23 May 2020 17:03:34 +0900 Subject: [PATCH 3/4] Fixed typo in a comment. --- .../spark/deploy/history/RealBrowserUIHistoryServerSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala index a24c7b8d8ae55..5c17ecffd493f 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala @@ -33,7 +33,7 @@ import org.apache.spark.internal.config.Tests.IS_TESTING import org.apache.spark.util.{ResetSystemProperties, Utils} /** - * Tests for HistoryServer with read web browsers. + * Tests for HistoryServer with real web browsers. */ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) extends SparkFunSuite with WebBrowser with Matchers with BeforeAndAfterAll From 4db2833813302bf8c2e79ebb40d19de8002beb68 Mon Sep 17 00:00:00 2001 From: Kousuke Saruta Date: Sat, 23 May 2020 17:10:44 +0900 Subject: [PATCH 4/4] Fixed style. --- .../deploy/history/RealBrowserUIHistoryServerSuite.scala | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala index 5c17ecffd493f..8a1e22c694497 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/RealBrowserUIHistoryServerSuite.scala @@ -22,7 +22,6 @@ import javax.servlet.http.HttpServletRequest import org.eclipse.jetty.proxy.ProxyServlet import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.openqa.selenium.WebDriver -import org.openqa.selenium.htmlunit.HtmlUnitDriver import org.scalatest._ import org.scalatestplus.selenium.WebBrowser @@ -49,7 +48,7 @@ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) private var server: HistoryServer = null private var port: Int = -1 - override def beforeAll() { + override def beforeAll(): Unit = { super.beforeAll() assume( sys.props(driverPropPrefix + driverProp) !== null, @@ -58,7 +57,7 @@ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) sys.props(driverProp) = sys.props(driverPropPrefix + driverProp) } - override def beforeEach { + override def beforeEach(): Unit = { super.beforeEach() if (server == null) { init() @@ -127,8 +126,6 @@ abstract class RealBrowserUIHistoryServerSuite(val driverProp: String) contextHandler.addServlet(holder, "/") server.attachHandler(contextHandler) - implicit val webDriver: WebDriver = new HtmlUnitDriver(true) - try { val url = s"http://localhost:$port"