diff --git a/core/src/main/resources/org/apache/spark/ui/static/historypage.js b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
index 5b9afb59ef8e288228cbc9a43a1d7cd093297230..c8094005c65dd526128a536c3efce52c3e64addb 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/historypage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/historypage.js
@@ -15,6 +15,12 @@
* limitations under the License.
*/
+var appLimit = -1;
+
+function setAppLimit(val) {
+ appLimit = val;
+}
+
function makeIdNumeric(id) {
var strs = id.split("_");
if (strs.length < 3) {
@@ -89,7 +95,7 @@ $(document).ready(function() {
requestedIncomplete = getParameterByName("showIncomplete", searchString);
requestedIncomplete = (requestedIncomplete == "true" ? true : false);
- $.getJSON("api/v1/applications", function(response,status,jqXHR) {
+ $.getJSON("api/v1/applications?limit=" + appLimit, function(response,status,jqXHR) {
var array = [];
var hasMultipleAttempts = false;
for (i in response) {
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
index 74f78021ed6cd8f8b2f75e2f3305ae2064e085b2..b4f5a6114f3decb5e07b2d3b8e16cf13af9accaf 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryPage.scala
@@ -45,7 +45,8 @@ private[history] class HistoryPage(parent: HistoryServer) extends WebUIPage("")
<script src={UIUtils.prependBaseUri("/static/dataTables.rowsGroup.js")}></script> ++
<div id="history-summary" class="span12 pagination"></div> ++
<script src={UIUtils.prependBaseUri("/static/utils.js")}></script> ++
- <script src={UIUtils.prependBaseUri("/static/historypage.js")}></script>
+ <script src={UIUtils.prependBaseUri("/static/historypage.js")}></script> ++
+ <script>setAppLimit({parent.maxApplications})</script>
} else if (requestedIncomplete) {
<h4>No incomplete applications found!</h4>
} else {
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
index d821474bdb5909f7dd20fcb8d609814a141fc7f1..c178917d8da3b4e72974d6f818276381ff74255e 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/HistoryServer.scala
@@ -28,6 +28,7 @@ import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
import org.apache.spark.{SecurityManager, SparkConf}
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.internal.Logging
+import org.apache.spark.internal.config._
import org.apache.spark.status.api.v1.{ApiRootResource, ApplicationInfo, ApplicationsListResource, UIRoot}
import org.apache.spark.ui.{SparkUI, UIUtils, WebUI}
import org.apache.spark.ui.JettyUtils._
@@ -55,6 +56,9 @@ class HistoryServer(
// How many applications to retain
private val retainedApplications = conf.getInt("spark.history.retainedApplications", 50)
+ // How many applications the summary ui displays
+ private[history] val maxApplications = conf.get(HISTORY_UI_MAX_APPS);
+
// application
private val appCache = new ApplicationCache(this, retainedApplications, new SystemClock())
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index ebce07c1e3b3eefd995b91990c1de86785695120..02d7d182a48c21977ed41d70141fd5aaffee4107 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -120,6 +120,10 @@ package object config {
.intConf
.createWithDefault(100000)
+ // To limit how many applications are shown in the History Server summary ui
+ private[spark] val HISTORY_UI_MAX_APPS =
+ ConfigBuilder("spark.history.ui.maxApplications").intConf.createWithDefault(Integer.MAX_VALUE)
+
private[spark] val IO_ENCRYPTION_ENABLED = ConfigBuilder("spark.io.encryption.enabled")
.booleanConf
.createWithDefault(false)
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
index 02fd2985fa20de4c573dc1a6418957b3b8ce1137..075b9ba37dc841cf4d40de02ce5f10af352b260c 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
@@ -29,7 +29,8 @@ private[v1] class ApplicationListResource(uiRoot: UIRoot) {
def appList(
@QueryParam("status") status: JList[ApplicationStatus],
@DefaultValue("2010-01-01") @QueryParam("minDate") minDate: SimpleDateParam,
- @DefaultValue("3000-01-01") @QueryParam("maxDate") maxDate: SimpleDateParam)
+ @DefaultValue("3000-01-01") @QueryParam("maxDate") maxDate: SimpleDateParam,
+ @QueryParam("limit") limit: Integer)
: Iterator[ApplicationInfo] = {
val allApps = uiRoot.getApplicationInfoList
val adjStatus = {
@@ -41,7 +42,7 @@ private[v1] class ApplicationListResource(uiRoot: UIRoot) {
}
val includeCompleted = adjStatus.contains(ApplicationStatus.COMPLETED)
val includeRunning = adjStatus.contains(ApplicationStatus.RUNNING)
- allApps.filter { app =>
+ val appList = allApps.filter { app =>
val anyRunning = app.attempts.exists(!_.completed)
// if any attempt is still running, we consider the app to also still be running
val statusOk = (!anyRunning && includeCompleted) ||
@@ -53,6 +54,11 @@ private[v1] class ApplicationListResource(uiRoot: UIRoot) {
}
statusOk && dateOk
}
+ if (limit != null) {
+ appList.take(limit)
+ } else {
+ appList
+ }
}
}
diff --git a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
new file mode 100644
index 0000000000000000000000000000000000000000..9165f549d7d25a102ccf1fa9213e0cc02d4d103f
--- /dev/null
+++ b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
@@ -0,0 +1,67 @@
+[ {
+ "id" : "local-1430917381534",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:11.398GMT",
+ "lastUpdated" : "",
+ "duration" : 10505,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
+ "id" : "local-1430917381535",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-05-06T13:03:00.893GMT",
+ "endTime" : "2015-05-06T13:03:00.950GMT",
+ "lastUpdated" : "",
+ "duration" : 57,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950,
+ "lastUpdatedEpoch" : 0
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-05-06T13:03:00.880GMT",
+ "endTime" : "2015-05-06T13:03:00.890GMT",
+ "lastUpdated" : "",
+ "duration" : 10,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890,
+ "lastUpdatedEpoch" : 0
+ } ]
+}, {
+ "id" : "local-1426533911241",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "attemptId" : "2",
+ "startTime" : "2015-03-17T23:11:50.242GMT",
+ "endTime" : "2015-03-17T23:12:25.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0
+ }, {
+ "attemptId" : "1",
+ "startTime" : "2015-03-16T19:25:10.242GMT",
+ "endTime" : "2015-03-16T19:25:45.177GMT",
+ "lastUpdated" : "",
+ "duration" : 34935,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0
+ } ]
+} ]
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 631a7cd9d5d7a4be4e25016044f3d69b459e3c8a..ae3f5d9c012ea2c97d57434c8d8ad3da43698920 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -100,6 +100,7 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
"minDate app list json" -> "applications?minDate=2015-02-10",
"maxDate app list json" -> "applications?maxDate=2015-02-10",
"maxDate2 app list json" -> "applications?maxDate=2015-02-03T16:42:40.000GMT",
+ "limit app list json" -> "applications?limit=3",
"one app json" -> "applications/local-1422981780767",
"one app multi-attempt json" -> "applications/local-1426533911241",
"job list json" -> "applications/local-1422981780767/jobs",
diff --git a/docs/monitoring.md b/docs/monitoring.md
index 6fdf87b4be57cbdf46268ed32da0ca5d625b033c..5804e4f26cd93ac383f8ae862a0f54b85b34421b 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -114,8 +114,17 @@ The history server can be configured as follows:
<td>spark.history.retainedApplications</td>
<td>50</td>
<td>
- The number of application UIs to retain. If this cap is exceeded, then the oldest
- applications will be removed.
+ The number of applications to retain UI data for in the cache. If this cap is exceeded, then
+ the oldest applications will be removed from the cache. If an application is not in the cache,
+ it will have to be loaded from disk if its accessed from the UI.
+ </td>
+ </tr>
+ <tr>
+ <td>spark.history.ui.maxApplications</td>
+ <td>Int.MaxValue</td>
+ <td>
+ The number of applications to display on the history summary page. Application UIs are still
+ available by accessing their URLs directly even if they are not displayed on the history summary page.
</td>
</tr>
<tr>
@@ -242,7 +251,8 @@ can be identified by their `[attempt-id]`. In the API listed below, when running
<br>Examples:
<br><code>?minDate=2015-02-10</code>
<br><code>?minDate=2015-02-03T16:42:40.000GMT</code>
- <br><code>?maxDate=[date]</code> latest date/time to list; uses same format as <code>minDate</code>.</td>
+ <br><code>?maxDate=[date]</code> latest date/time to list; uses same format as <code>minDate</code>.
+ <br><code>?limit=[limit]</code> limits the number of applications listed.</td>
</tr>
<tr>
<td><code>/applications/[app-id]/jobs</code></td>