Repository: spark Updated Branches: refs/heads/master 1974d1d34 -> 65b75e66e
[SPARK-13776][WEBUI] Limit the max number of acceptors and selectors for Jetty ## What changes were proposed in this pull request? As each acceptor/selector in Jetty will use one thread, the number of threads should at least be the number of acceptors and selectors plus 1. Otherwise, the thread pool of Jetty server may be exhausted by acceptors/selectors and not be able to response any request. To avoid wasting threads, the PR limits the max number of acceptors and selectors and also updates the max thread number if necessary. ## How was this patch tested? Just make sure we don't break any existing tests Author: Shixiong Zhu <[email protected]> Closes #11615 from zsxwing/SPARK-13776. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/65b75e66 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/65b75e66 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/65b75e66 Branch: refs/heads/master Commit: 65b75e66e89075e23ccaa3813648397b3ec010fe Parents: 1974d1d Author: Shixiong Zhu <[email protected]> Authored: Thu Mar 17 13:05:29 2016 +0000 Committer: Sean Owen <[email protected]> Committed: Thu Mar 17 13:05:29 2016 +0000 ---------------------------------------------------------------------- .../scala/org/apache/spark/ui/JettyUtils.scala | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/65b75e66/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala index 31312fb..d9fecc5 100644 --- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala +++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala @@ -25,7 +25,7 @@ import scala.collection.mutable.ArrayBuffer import scala.language.implicitConversions import scala.xml.Node -import org.eclipse.jetty.server.{Connector, Request, Server} +import org.eclipse.jetty.server.{AbstractConnector, Connector, Request, Server} import org.eclipse.jetty.server.handler._ import org.eclipse.jetty.server.nio.SelectChannelConnector import org.eclipse.jetty.server.ssl.SslSelectChannelConnector @@ -271,9 +271,24 @@ private[spark] object JettyUtils extends Logging { gzipHandlers.foreach(collection.addHandler) connectors.foreach(_.setHost(hostName)) + // As each acceptor and each selector will use one thread, the number of threads should at + // least be the number of acceptors and selectors plus 1. (See SPARK-13776) + var minThreads = 1 + connectors.foreach { c => + // Currently we only use "SelectChannelConnector" + val connector = c.asInstanceOf[SelectChannelConnector] + // Limit the max acceptor number to 8 so that we don't waste a lot of threads + connector.setAcceptors(math.min(connector.getAcceptors, 8)) + // The number of selectors always equals to the number of acceptors + minThreads += connector.getAcceptors * 2 + } server.setConnectors(connectors.toArray) val pool = new QueuedThreadPool + if (serverName.nonEmpty) { + pool.setName(serverName) + } + pool.setMaxThreads(math.max(pool.getMaxThreads, minThreads)) pool.setDaemon(true) server.setThreadPool(pool) val errorHandler = new ErrorHandler() --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
