From 88a536babf119b7e331d02aac5d52b57658803bf Mon Sep 17 00:00:00 2001
From: IngoSchuster <ingo.schuster@de.ibm.com>
Date: Fri, 30 Jun 2017 11:16:09 +0800
Subject: [PATCH] [SPARK-21176][WEB UI] Limit number of selector threads for
admin ui proxy servlets to 8
## What changes were proposed in this pull request?
Please see also https://issues.apache.org/jira/browse/SPARK-21176
This change limits the number of selector threads that jetty creates to maximum 8 per proxy servlet (Jetty default is number of processors / 2).
The newHttpClient for Jettys ProxyServlet class is overwritten to avoid the Jetty defaults (which are designed for high-performance http servers).
Once https://github.com/eclipse/jetty.project/issues/1643 is available, the code could be cleaned up to avoid the method override.
I really need this on v2.1.1 - what is the best way for a backport automatic merge works fine)? Shall I create another PR?
## How was this patch tested?
(Please explain how this patch was tested. E.g. unit tests, integration tests, manual tests)
The patch was tested manually on a Spark cluster with a head node that has 88 processors using JMX to verify that the number of selector threads is now limited to 8 per proxy.
gurvindersingh zsxwing can you please review the change?
Author: IngoSchuster <ingo.schuster@de.ibm.com>
Author: Ingo Schuster <ingo.schuster@de.ibm.com>
Closes #18437 from IngoSchuster/master.
---
.../main/scala/org/apache/spark/ui/JettyUtils.scala | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
index edf328b5ae..b9371c7ad7 100644
--- a/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/JettyUtils.scala
@@ -26,6 +26,8 @@ import scala.language.implicitConversions
import scala.xml.Node
import org.eclipse.jetty.client.api.Response
+import org.eclipse.jetty.client.HttpClient
+import org.eclipse.jetty.client.http.HttpClientTransportOverHTTP
import org.eclipse.jetty.proxy.ProxyServlet
import org.eclipse.jetty.server._
import org.eclipse.jetty.server.handler._
@@ -208,6 +210,16 @@ private[spark] object JettyUtils extends Logging {
rewrittenURI.toString()
}
+ override def newHttpClient(): HttpClient = {
+ // SPARK-21176: Use the Jetty logic to calculate the number of selector threads (#CPUs/2),
+ // but limit it to 8 max.
+ // Otherwise, it might happen that we exhaust the threadpool since in reverse proxy mode
+ // a proxy is instantiated for each executor. If the head node has many processors, this
+ // can quickly add up to an unreasonably high number of threads.
+ val numSelectors = math.max(1, math.min(8, Runtime.getRuntime().availableProcessors() / 2))
+ new HttpClient(new HttpClientTransportOverHTTP(numSelectors), null)
+ }
+
override def filterServerResponseHeader(
clientRequest: HttpServletRequest,
serverResponse: Response,
--
GitLab