Repository: spark Updated Branches: refs/heads/branch-2.0 9abff1bcd -> de545e7c8
[SPARK-16808][CORE] History Server main page does not honor APPLICATION_WEB_PROXY_BASE ## What changes were proposed in this pull request? Backport SPARK-16808 (https://github.com/apache/spark/pull/15742) to branch-2.0. Application links generated on the history server UI no longer (regression from 1.6) contain the configured spark.ui.proxyBase in the links. To address this, made the uiRoot available globally to all javascripts for Web UI. Updated the mustache template (historypage-template.html) to include the uiroot for rendering links to the applications. The existing test was not sufficient to verify the scenario where ajax call is used to populate the application listing template, so added a new selenium test case to cover this scenario. ## How was this patch tested? Existing tests and a new unit test. No visual changes to the UI. Author: Vinayak <[email protected]> Closes #15855 from vijoshi/SPARK-16808_branch-2.0. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/de545e7c Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/de545e7c Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/de545e7c Branch: refs/heads/branch-2.0 Commit: de545e7c877a71028e1b3d3bfbea127f35cfc2d8 Parents: 9abff1b Author: Vinayak <[email protected]> Authored: Tue Nov 15 13:08:38 2016 -0800 Committer: Marcelo Vanzin <[email protected]> Committed: Tue Nov 15 13:08:38 2016 -0800 ---------------------------------------------------------------------- core/pom.xml | 5 ++ .../spark/ui/static/historypage-template.html | 4 +- .../org/apache/spark/ui/static/historypage.js | 6 +- .../org/apache/spark/ui/static/webui.js | 22 ++++++ .../scala/org/apache/spark/ui/UIUtils.scala | 3 +- .../deploy/history/HistoryServerSuite.scala | 80 +++++++++++++++++++- pom.xml | 6 ++ 7 files changed, 119 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/de545e7c/core/pom.xml ---------------------------------------------------------------------- diff --git a/core/pom.xml b/core/pom.xml index 55cbe89..837f889 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -273,6 +273,11 @@ <version>${oro.version}</version> </dependency> <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-proxy</artifactId> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.seleniumhq.selenium</groupId> <artifactId>selenium-java</artifactId> <scope>test</scope> http://git-wip-us.apache.org/repos/asf/spark/blob/de545e7c/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html index a2b3826..4b86441 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage-template.html @@ -64,10 +64,10 @@ <tbody> {{#applications}} <tr> - <td class="rowGroupColumn"><span title="{{id}}"><a href="/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td> + <td class="rowGroupColumn"><span title="{{id}}"><a href="{{uiroot}}/history/{{id}}/{{num}}/jobs/">{{id}}</a></span></td> <td class="rowGroupColumn">{{name}}</td> {{#attempts}} - <td class="attemptIDSpan"><a href="/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td> + <td class="attemptIDSpan"><a href="{{uiroot}}/history/{{id}}/{{attemptId}}/jobs/">{{attemptId}}</a></td> <td>{{startTime}}</td> <td>{{endTime}}</td> <td><span title="{{duration}}" class="durationClass">{{duration}}</span></td> http://git-wip-us.apache.org/repos/asf/spark/blob/de545e7c/core/src/main/resources/org/apache/spark/ui/static/historypage.js ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js index 177120a..ae6d41f 100644 --- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js +++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js @@ -141,7 +141,11 @@ $(document).ready(function() { } } - var data = {"applications": array} + var data = { + "uiroot": uiRoot, + "applications": array + } + $.get("static/historypage-template.html", function(template) { historySummary.append(Mustache.render($(template).filter("#history-summary-template").html(),data)); var selector = "#history-summary-table"; http://git-wip-us.apache.org/repos/asf/spark/blob/de545e7c/core/src/main/resources/org/apache/spark/ui/static/webui.js ---------------------------------------------------------------------- diff --git a/core/src/main/resources/org/apache/spark/ui/static/webui.js b/core/src/main/resources/org/apache/spark/ui/static/webui.js new file mode 100644 index 0000000..eb0b748 --- /dev/null +++ b/core/src/main/resources/org/apache/spark/ui/static/webui.js @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +var uiRoot = ""; + +function setUIRoot(val) { + uiRoot = val; +} http://git-wip-us.apache.org/repos/asf/spark/blob/de545e7c/core/src/main/scala/org/apache/spark/ui/UIUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala index 1aa85d6..f7d37f6 100644 --- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala @@ -168,7 +168,8 @@ private[spark] object UIUtils extends Logging { <script src={prependBaseUri("/static/table.js")}></script> <script src={prependBaseUri("/static/additional-metrics.js")}></script> <script src={prependBaseUri("/static/timeline-view.js")}></script> - <script src={prependBaseUri("/static/log-view.js")}></script> + <script src={prependBaseUri("/static/webui.js")}></script> + <script>setUIRoot('{UIUtils.uiRoot}')</script> } def vizHeaderNodes: Seq[Node] = { http://git-wip-us.apache.org/repos/asf/spark/blob/de545e7c/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala ---------------------------------------------------------------------- diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala index ae3f5d9..51efdd0 100644 --- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala +++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala @@ -29,6 +29,8 @@ import com.codahale.metrics.Counter import com.google.common.io.{ByteStreams, Files} import org.apache.commons.io.{FileUtils, IOUtils} import org.apache.hadoop.fs.{FileStatus, FileSystem, Path} +import org.eclipse.jetty.proxy.ProxyServlet +import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder} import org.json4s.JsonAST._ import org.json4s.jackson.JsonMethods import org.json4s.jackson.JsonMethods._ @@ -258,8 +260,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers getContentAndCode("foobar")._1 should be (HttpServletResponse.SC_NOT_FOUND) } - test("relative links are prefixed with uiRoot (spark.ui.proxyBase)") { - val proxyBaseBeforeTest = System.getProperty("spark.ui.proxyBase") + test("static relative links are prefixed with uiRoot (spark.ui.proxyBase)") { val uiRoot = Option(System.getenv("APPLICATION_WEB_PROXY_BASE")).getOrElse("/testwebproxybase") val page = new HistoryPage(server) val request = mock[HttpServletRequest] @@ -267,7 +268,6 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers // when System.setProperty("spark.ui.proxyBase", uiRoot) val response = page.render(request) - System.setProperty("spark.ui.proxyBase", Option(proxyBaseBeforeTest).getOrElse("")) // then val urls = response \\ "@href" map (_.toString) @@ -275,6 +275,80 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers all (siteRelativeLinks) should startWith (uiRoot) } + test("ajax rendered relative links are prefixed with uiRoot (spark.ui.proxyBase)") { + val uiRoot = "/testwebproxybase" + System.setProperty("spark.ui.proxyBase", uiRoot) + + server.stop() + + val conf = new SparkConf() + .set("spark.history.fs.logDirectory", logDir.getAbsolutePath) + .set("spark.history.fs.update.interval", "0") + .set("spark.testing", "true") + + provider = new FsHistoryProvider(conf) + provider.checkForLogs() + val securityManager = new SecurityManager(conf) + + server = new HistoryServer(conf, provider, securityManager, 18080) + server.initialize() + server.bind() + + val port = server.boundPort + + val servlet = new ProxyServlet { + override def rewriteTarget(request: HttpServletRequest): String = { + // servlet acts like a proxy that redirects calls made on + // spark.ui.proxyBase context path to the normal servlet handlers operating off "/" + val sb = request.getRequestURL() + + if (request.getQueryString() != null) { + sb.append(s"?${request.getQueryString()}") + } + + val proxyidx = sb.indexOf(uiRoot) + sb.delete(proxyidx, proxyidx + uiRoot.length).toString + } + } + + val contextHandler = new ServletContextHandler + val holder = new ServletHolder(servlet) + contextHandler.setContextPath(uiRoot) + contextHandler.addServlet(holder, "/") + server.attachHandler(contextHandler) + + implicit val webDriver: WebDriver = new HtmlUnitDriver(true) { + getWebClient.getOptions.setThrowExceptionOnScriptError(false) + } + + try { + val url = s"http://localhost:$port"; + + go to s"$url$uiRoot" + + // expect the ajax call to finish in 5 seconds + implicitlyWait(org.scalatest.time.Span(5, org.scalatest.time.Seconds)) + + // once this findAll call returns, we know the ajax load of the table completed + findAll(ClassNameQuery("odd")) + + val links = findAll(TagNameQuery("a")) + .map(_.attribute("href")) + .filter(_.isDefined) + .map(_.get) + .filter(_.startsWith(url)).toList + + // there are atleast some URL links that were generated via javascript, + // and they all contain the spark.ui.proxyBase (uiRoot) + links.length should be > 4 + all(links) should startWith(url + uiRoot) + } finally { + contextHandler.stop() + quit() + } + + } + test("incomplete apps get refreshed") { implicit val webDriver: WebDriver = new HtmlUnitDriver http://git-wip-us.apache.org/repos/asf/spark/blob/de545e7c/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index d97f348..ac2ec0e 100644 --- a/pom.xml +++ b/pom.xml @@ -432,6 +432,12 @@ <version>${commons.httpcore.version}</version> </dependency> <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-proxy</artifactId> + <version>${jetty.version}</version> + <scope>test</scope> + </dependency> + <dependency> <groupId>org.seleniumhq.selenium</groupId> <artifactId>selenium-java</artifactId> <version>${selenium.version}</version> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
