This is an automated email from the ASF dual-hosted git repository.
gengliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 92877c4 [SPARK-31765][WEBUI] Upgrade HtmlUnit >= 2.37.0
92877c4 is described below
commit 92877c4ef2ad113c156b7d9c359f396187c78fa3
Author: Kousuke Saruta <[email protected]>
AuthorDate: Thu May 21 11:43:25 2020 -0700
[SPARK-31765][WEBUI] Upgrade HtmlUnit >= 2.37.0
### What changes were proposed in this pull request?
This PR upgrades HtmlUnit.
Selenium and Jetty also upgraded because of dependency.
### Why are the changes needed?
Recently, a security issue which affects HtmlUnit is reported.
https://nvd.nist.gov/vuln/detail/CVE-2020-5529
According to the report, arbitrary code can be run by malicious users.
HtmlUnit is used for test so the impact might not be large but it's better
to upgrade it just in case.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Existing testcases.
Closes #28585 from sarutak/upgrade-htmlunit.
Authored-by: Kousuke Saruta <[email protected]>
Signed-off-by: Gengliang Wang <[email protected]>
---
core/pom.xml | 2 +-
core/src/main/scala/org/apache/spark/ui/JettyUtils.scala | 7 ++++++-
core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala | 3 ++-
pom.xml | 10 +++++-----
sql/core/pom.xml | 2 +-
sql/hive-thriftserver/pom.xml | 2 +-
streaming/pom.xml | 2 +-
7 files changed, 17 insertions(+), 11 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index b0f6888..14b217d 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -334,7 +334,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
<!-- Coerce sbt into honoring these dependency updates: -->
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index 4b4788f..f1962ef 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -23,6 +23,7 @@ import javax.servlet.DispatcherType
import javax.servlet.http._
import scala.language.implicitConversions
+import scala.util.Try
import scala.xml.Node
import org.eclipse.jetty.client.HttpClient
@@ -500,7 +501,11 @@ private[spark] case class ServerInfo(
threadPool match {
case pool: QueuedThreadPool =>
// Workaround for SPARK-30385 to avoid Jetty's acceptor thread shrink.
- pool.setIdleTimeout(0)
+ // As of Jetty 9.4.21, the implementation of
+ // QueuedThreadPool#setIdleTimeout is changed and IllegalStateException
+ // will be thrown if we try to set idle timeout after the server has
started.
+ // But this workaround works for Jetty 9.4.28 by ignoring the
exception.
+ Try(pool.setIdleTimeout(0))
case _ =>
}
server.stop()
diff --git a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
index 3ec9385..e96d82a 100644
--- a/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
+++ b/core/src/test/scala/org/apache/spark/ui/UISeleniumSuite.scala
@@ -24,6 +24,7 @@ import javax.servlet.http.{HttpServletRequest,
HttpServletResponse}
import scala.io.Source
import scala.xml.Node
+import com.gargoylesoftware.css.parser.CSSParseException
import com.gargoylesoftware.htmlunit.DefaultCssErrorHandler
import org.json4s._
import org.json4s.jackson.JsonMethods
@@ -33,7 +34,6 @@ import org.scalatest._
import org.scalatest.concurrent.Eventually._
import org.scalatest.time.SpanSugar._
import org.scalatestplus.selenium.WebBrowser
-import org.w3c.css.sac.CSSParseException
import org.apache.spark._
import org.apache.spark.LocalSparkContext._
@@ -784,6 +784,7 @@ class UISeleniumSuite extends SparkFunSuite with WebBrowser
with Matchers with B
eventually(timeout(10.seconds), interval(50.milliseconds)) {
goToUi(sc, "/jobs")
+
val jobDesc =
driver.findElement(By.cssSelector("div[class='application-timeline-content']"))
jobDesc.getAttribute("data-title") should include ("collect at
<console>:25")
diff --git a/pom.xml b/pom.xml
index fd4cebc..29f7fec 100644
--- a/pom.xml
+++ b/pom.xml
@@ -139,7 +139,7 @@
<orc.classifier></orc.classifier>
<hive.parquet.group>com.twitter</hive.parquet.group>
<hive.parquet.version>1.6.0</hive.parquet.version>
- <jetty.version>9.4.18.v20190429</jetty.version>
+ <jetty.version>9.4.28.v20200408</jetty.version>
<javaxservlet.version>3.1.0</javaxservlet.version>
<chill.version>0.9.5</chill.version>
<ivy.version>2.4.0</ivy.version>
@@ -187,8 +187,8 @@
<libthrift.version>0.12.0</libthrift.version>
<antlr4.version>4.7.1</antlr4.version>
<jpam.version>1.1</jpam.version>
- <selenium.version>2.52.0</selenium.version>
- <htmlunit.version>2.22</htmlunit.version>
+ <selenium.version>3.141.59</selenium.version>
+ <htmlunit.version>2.40.0</htmlunit.version>
<!--
Managed up from older version from Avro; sync with
jackson-module-paranamer dependency version
-->
@@ -591,8 +591,8 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
- <version>${selenium.version}</version>
+ <artifactId>htmlunit-driver</artifactId>
+ <version>${htmlunit.version}</version>
<scope>test</scope>
</dependency>
<!-- Update htmlunit dependency that selenium uses for better JS support
-->
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 7c5fcba..e4ef146 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -162,7 +162,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
</dependencies>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 1de2677..5bf20b2 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -95,7 +95,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index ea351d4..53b49dd 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -109,7 +109,7 @@
</dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
- <artifactId>selenium-htmlunit-driver</artifactId>
+ <artifactId>htmlunit-driver</artifactId>
<scope>test</scope>
</dependency>
<dependency>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
