diff --git a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
index 930a0698928d1e51d666a8d0271be332e655dd8b..cb9922d23c445cd910e45cca9471ce9122dbbc9f 100644
--- a/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+++ b/core/src/main/resources/org/apache/spark/ui/static/executorspage.js
@@ -253,10 +253,14 @@ $(document).ready(function () {
var deadTotalBlacklisted = 0;
response.forEach(function (exec) {
- exec.onHeapMemoryUsed = exec.hasOwnProperty('onHeapMemoryUsed') ? exec.onHeapMemoryUsed : 0;
- exec.maxOnHeapMemory = exec.hasOwnProperty('maxOnHeapMemory') ? exec.maxOnHeapMemory : 0;
- exec.offHeapMemoryUsed = exec.hasOwnProperty('offHeapMemoryUsed') ? exec.offHeapMemoryUsed : 0;
- exec.maxOffHeapMemory = exec.hasOwnProperty('maxOffHeapMemory') ? exec.maxOffHeapMemory : 0;
+ var memoryMetrics = {
+ usedOnHeapStorageMemory: 0,
+ usedOffHeapStorageMemory: 0,
+ totalOnHeapStorageMemory: 0,
+ totalOffHeapStorageMemory: 0
+ };
+
+ exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
});
response.forEach(function (exec) {
@@ -264,10 +268,10 @@ $(document).ready(function () {
allRDDBlocks += exec.rddBlocks;
allMemoryUsed += exec.memoryUsed;
allMaxMemory += exec.maxMemory;
- allOnHeapMemoryUsed += exec.onHeapMemoryUsed;
- allOnHeapMaxMemory += exec.maxOnHeapMemory;
- allOffHeapMemoryUsed += exec.offHeapMemoryUsed;
- allOffHeapMaxMemory += exec.maxOffHeapMemory;
+ allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+ allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+ allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+ allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
allDiskUsed += exec.diskUsed;
allTotalCores += exec.totalCores;
allMaxTasks += exec.maxTasks;
@@ -286,10 +290,10 @@ $(document).ready(function () {
activeRDDBlocks += exec.rddBlocks;
activeMemoryUsed += exec.memoryUsed;
activeMaxMemory += exec.maxMemory;
- activeOnHeapMemoryUsed += exec.onHeapMemoryUsed;
- activeOnHeapMaxMemory += exec.maxOnHeapMemory;
- activeOffHeapMemoryUsed += exec.offHeapMemoryUsed;
- activeOffHeapMaxMemory += exec.maxOffHeapMemory;
+ activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+ activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+ activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+ activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
activeDiskUsed += exec.diskUsed;
activeTotalCores += exec.totalCores;
activeMaxTasks += exec.maxTasks;
@@ -308,10 +312,10 @@ $(document).ready(function () {
deadRDDBlocks += exec.rddBlocks;
deadMemoryUsed += exec.memoryUsed;
deadMaxMemory += exec.maxMemory;
- deadOnHeapMemoryUsed += exec.onHeapMemoryUsed;
- deadOnHeapMaxMemory += exec.maxOnHeapMemory;
- deadOffHeapMemoryUsed += exec.offHeapMemoryUsed;
- deadOffHeapMaxMemory += exec.maxOffHeapMemory;
+ deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
+ deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
+ deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
+ deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
deadDiskUsed += exec.diskUsed;
deadTotalCores += exec.totalCores;
deadMaxTasks += exec.maxTasks;
@@ -431,10 +435,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
- return row.onHeapMemoryUsed;
+ return row.memoryMetrics.usedOnHeapStorageMemory;
else
- return (formatBytes(row.onHeapMemoryUsed, type) + ' / ' +
- formatBytes(row.maxOnHeapMemory, type));
+ return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
+ formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('on_heap_memory')
@@ -443,10 +447,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
- return row.offHeapMemoryUsed;
+ return row.memoryMetrics.usedOffHeapStorageMemory;
else
- return (formatBytes(row.offHeapMemoryUsed, type) + ' / ' +
- formatBytes(row.maxOffHeapMemory, type));
+ return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
+ formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('off_heap_memory')
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index d159b9450ef5cb22145c2ca2f63d1980e190fc3f..56d8e51732ffd04fe0693e48b278ca9baaf23b66 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -76,10 +76,13 @@ class ExecutorSummary private[spark](
val isBlacklisted: Boolean,
val maxMemory: Long,
val executorLogs: Map[String, String],
- val onHeapMemoryUsed: Option[Long],
- val offHeapMemoryUsed: Option[Long],
- val maxOnHeapMemory: Option[Long],
- val maxOffHeapMemory: Option[Long])
+ val memoryMetrics: Option[MemoryMetrics])
+
+class MemoryMetrics private[spark](
+ val usedOnHeapStorageMemory: Long,
+ val usedOffHeapStorageMemory: Long,
+ val totalOnHeapStorageMemory: Long,
+ val totalOffHeapStorageMemory: Long)
class JobData private[spark](
val jobId: Int,
diff --git a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
index 0a3c63d14ca8a2fcf8825ea3a1e179492cea0fb0..b7cbed468517ca2872dc3f5a2028549a4ae3b9c0 100644
--- a/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
@@ -21,7 +21,7 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.Node
-import org.apache.spark.status.api.v1.ExecutorSummary
+import org.apache.spark.status.api.v1.{ExecutorSummary, MemoryMetrics}
import org.apache.spark.ui.{UIUtils, WebUIPage}
// This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive
@@ -114,10 +114,16 @@ private[spark] object ExecutorsPage {
val rddBlocks = status.numBlocks
val memUsed = status.memUsed
val maxMem = status.maxMem
- val onHeapMemUsed = status.onHeapMemUsed
- val offHeapMemUsed = status.offHeapMemUsed
- val maxOnHeapMem = status.maxOnHeapMem
- val maxOffHeapMem = status.maxOffHeapMem
+ val memoryMetrics = for {
+ onHeapUsed <- status.onHeapMemUsed
+ offHeapUsed <- status.offHeapMemUsed
+ maxOnHeap <- status.maxOnHeapMem
+ maxOffHeap <- status.maxOffHeapMem
+ } yield {
+ new MemoryMetrics(onHeapUsed, offHeapUsed, maxOnHeap, maxOffHeap)
+ }
+
+
val diskUsed = status.diskUsed
val taskSummary = listener.executorToTaskSummary.getOrElse(execId, ExecutorTaskSummary(execId))
@@ -142,10 +148,7 @@ private[spark] object ExecutorsPage {
taskSummary.isBlacklisted,
maxMem,
taskSummary.executorLogs,
- onHeapMemUsed,
- offHeapMemUsed,
- maxOnHeapMem,
- maxOffHeapMem
+ memoryMetrics
)
}
}
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
index e732af2663503081ece27f0c7757df058443c735..0f94e3b255dbc33e6de10d06c2cf8b573464c21b 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
@@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
@@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
@@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},
-
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
@@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
@@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
index e732af2663503081ece27f0c7757df058443c735..0f94e3b255dbc33e6de10d06c2cf8b573464c21b 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
@@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
@@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
@@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},
-
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
@@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
@@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
- "onHeapMemoryUsed" : 0,
- "offHeapMemoryUsed" : 0,
- "maxOnHeapMemory" : 384093388,
- "maxOffHeapMemory" : 524288000
+ "memoryMetrics": {
+ "usedOnHeapStorageMemory": 0,
+ "usedOffHeapStorageMemory": 0,
+ "totalOnHeapStorageMemory": 384093388,
+ "totalOffHeapStorageMemory": 524288000
+ }
} ]