Skip to content
Snippets Groups Projects
Commit 66dd5b83 authored by jerryshao's avatar jerryshao Committed by Imran Rashid
Browse files

[SPARK-20391][CORE] Rename memory related fields in ExecutorSummay 

## What changes were proposed in this pull request?

This is a follow-up of #14617 to make the name of memory related fields more meaningful.

Here  for the backward compatibility, I didn't change `maxMemory` and `memoryUsed` fields.

## How was this patch tested?

Existing UT and local verification.

CC squito and tgravescs .

Author: jerryshao <sshao@hortonworks.com>

Closes #17700 from jerryshao/SPARK-20391.
parent dbb06c68
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 105 additions and 77 deletions
core/src/main/resources/org/apache/spark/ui/static/executorspage.js
+ 26
− 22
View file @ 66dd5b83
......@@ -253,10 +253,14 @@ $(document).ready(function () {
var deadTotalBlacklisted = 0;
response.forEach(function (exec) {
exec.onHeapMemoryUsed = exec.hasOwnProperty('onHeapMemoryUsed') ? exec.onHeapMemoryUsed : 0;
exec.maxOnHeapMemory = exec.hasOwnProperty('maxOnHeapMemory') ? exec.maxOnHeapMemory : 0;
exec.offHeapMemoryUsed = exec.hasOwnProperty('offHeapMemoryUsed') ? exec.offHeapMemoryUsed : 0;
exec.maxOffHeapMemory = exec.hasOwnProperty('maxOffHeapMemory') ? exec.maxOffHeapMemory : 0;
var memoryMetrics = {
usedOnHeapStorageMemory: 0,
usedOffHeapStorageMemory: 0,
totalOnHeapStorageMemory: 0,
totalOffHeapStorageMemory: 0
};
exec.memoryMetrics = exec.hasOwnProperty('memoryMetrics') ? exec.memoryMetrics : memoryMetrics;
});
response.forEach(function (exec) {
......@@ -264,10 +268,10 @@ $(document).ready(function () {
allRDDBlocks += exec.rddBlocks;
allMemoryUsed += exec.memoryUsed;
allMaxMemory += exec.maxMemory;
allOnHeapMemoryUsed += exec.onHeapMemoryUsed;
allOnHeapMaxMemory += exec.maxOnHeapMemory;
allOffHeapMemoryUsed += exec.offHeapMemoryUsed;
allOffHeapMaxMemory += exec.maxOffHeapMemory;
allOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
allOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
allOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
allOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
allDiskUsed += exec.diskUsed;
allTotalCores += exec.totalCores;
allMaxTasks += exec.maxTasks;
......@@ -286,10 +290,10 @@ $(document).ready(function () {
activeRDDBlocks += exec.rddBlocks;
activeMemoryUsed += exec.memoryUsed;
activeMaxMemory += exec.maxMemory;
activeOnHeapMemoryUsed += exec.onHeapMemoryUsed;
activeOnHeapMaxMemory += exec.maxOnHeapMemory;
activeOffHeapMemoryUsed += exec.offHeapMemoryUsed;
activeOffHeapMaxMemory += exec.maxOffHeapMemory;
activeOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
activeOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
activeOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
activeOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
activeDiskUsed += exec.diskUsed;
activeTotalCores += exec.totalCores;
activeMaxTasks += exec.maxTasks;
......@@ -308,10 +312,10 @@ $(document).ready(function () {
deadRDDBlocks += exec.rddBlocks;
deadMemoryUsed += exec.memoryUsed;
deadMaxMemory += exec.maxMemory;
deadOnHeapMemoryUsed += exec.onHeapMemoryUsed;
deadOnHeapMaxMemory += exec.maxOnHeapMemory;
deadOffHeapMemoryUsed += exec.offHeapMemoryUsed;
deadOffHeapMaxMemory += exec.maxOffHeapMemory;
deadOnHeapMemoryUsed += exec.memoryMetrics.usedOnHeapStorageMemory;
deadOnHeapMaxMemory += exec.memoryMetrics.totalOnHeapStorageMemory;
deadOffHeapMemoryUsed += exec.memoryMetrics.usedOffHeapStorageMemory;
deadOffHeapMaxMemory += exec.memoryMetrics.totalOffHeapStorageMemory;
deadDiskUsed += exec.diskUsed;
deadTotalCores += exec.totalCores;
deadMaxTasks += exec.maxTasks;
......@@ -431,10 +435,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
return row.onHeapMemoryUsed;
return row.memoryMetrics.usedOnHeapStorageMemory;
else
return (formatBytes(row.onHeapMemoryUsed, type) + ' / ' +
formatBytes(row.maxOnHeapMemory, type));
return (formatBytes(row.memoryMetrics.usedOnHeapStorageMemory, type) + ' / ' +
formatBytes(row.memoryMetrics.totalOnHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('on_heap_memory')
......@@ -443,10 +447,10 @@ $(document).ready(function () {
{
data: function (row, type) {
if (type !== 'display')
return row.offHeapMemoryUsed;
return row.memoryMetrics.usedOffHeapStorageMemory;
else
return (formatBytes(row.offHeapMemoryUsed, type) + ' / ' +
formatBytes(row.maxOffHeapMemory, type));
return (formatBytes(row.memoryMetrics.usedOffHeapStorageMemory, type) + ' / ' +
formatBytes(row.memoryMetrics.totalOffHeapStorageMemory, type));
},
"fnCreatedCell": function (nTd, sData, oData, iRow, iCol) {
$(nTd).addClass('off_heap_memory')
......
core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+ 7
− 4
View file @ 66dd5b83
......@@ -76,10 +76,13 @@ class ExecutorSummary private[spark](
val isBlacklisted: Boolean,
val maxMemory: Long,
val executorLogs: Map[String, String],
val onHeapMemoryUsed: Option[Long],
val offHeapMemoryUsed: Option[Long],
val maxOnHeapMemory: Option[Long],
val maxOffHeapMemory: Option[Long])
val memoryMetrics: Option[MemoryMetrics])
class MemoryMetrics private[spark](
val usedOnHeapStorageMemory: Long,
val usedOffHeapStorageMemory: Long,
val totalOnHeapStorageMemory: Long,
val totalOffHeapStorageMemory: Long)
class JobData private[spark](
val jobId: Int,
......
core/src/main/scala/org/apache/spark/ui/exec/ExecutorsPage.scala
+ 12
− 9
View file @ 66dd5b83
......@@ -21,7 +21,7 @@ import javax.servlet.http.HttpServletRequest
import scala.xml.Node
import org.apache.spark.status.api.v1.ExecutorSummary
import org.apache.spark.status.api.v1.{ExecutorSummary, MemoryMetrics}
import org.apache.spark.ui.{UIUtils, WebUIPage}
// This isn't even used anymore -- but we need to keep it b/c of a MiMa false positive
......@@ -114,10 +114,16 @@ private[spark] object ExecutorsPage {
val rddBlocks = status.numBlocks
val memUsed = status.memUsed
val maxMem = status.maxMem
val onHeapMemUsed = status.onHeapMemUsed
val offHeapMemUsed = status.offHeapMemUsed
val maxOnHeapMem = status.maxOnHeapMem
val maxOffHeapMem = status.maxOffHeapMem
val memoryMetrics = for {
onHeapUsed <- status.onHeapMemUsed
offHeapUsed <- status.offHeapMemUsed
maxOnHeap <- status.maxOnHeapMem
maxOffHeap <- status.maxOffHeapMem
} yield {
new MemoryMetrics(onHeapUsed, offHeapUsed, maxOnHeap, maxOffHeap)
}
val diskUsed = status.diskUsed
val taskSummary = listener.executorToTaskSummary.getOrElse(execId, ExecutorTaskSummary(execId))
......@@ -142,10 +148,7 @@ private[spark] object ExecutorsPage {
taskSummary.isBlacklisted,
maxMem,
taskSummary.executorLogs,
onHeapMemUsed,
offHeapMemUsed,
maxOnHeapMem,
maxOffHeapMem
memoryMetrics
)
}
}
core/src/test/resources/HistoryServerExpectations/executor_memory_usage_expectation.json
+ 30
− 21
View file @ 66dd5b83
......@@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
......@@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
......@@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
......@@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
......@@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
} ]
core/src/test/resources/HistoryServerExpectations/executor_node_blacklisting_expectation.json
+ 30
− 21
View file @ 66dd5b83
......@@ -22,10 +22,12 @@
"stdout" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stdout",
"stderr" : "http://172.22.0.167:51469/logPage/?appId=app-20161116163331-0000&executorId=2&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "driver",
"hostPort" : "172.22.0.167:51475",
......@@ -47,10 +49,12 @@
"isBlacklisted" : true,
"maxMemory" : 908381388,
"executorLogs" : { },
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "1",
"hostPort" : "172.22.0.167:51490",
......@@ -75,11 +79,12 @@
"stdout" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stdout",
"stderr" : "http://172.22.0.167:51467/logPage/?appId=app-20161116163331-0000&executorId=1&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "0",
"hostPort" : "172.22.0.167:51491",
......@@ -104,10 +109,12 @@
"stdout" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stdout",
"stderr" : "http://172.22.0.167:51465/logPage/?appId=app-20161116163331-0000&executorId=0&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
}, {
"id" : "3",
"hostPort" : "172.22.0.167:51485",
......@@ -132,8 +139,10 @@
"stdout" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stdout",
"stderr" : "http://172.22.0.167:51466/logPage/?appId=app-20161116163331-0000&executorId=3&logType=stderr"
},
"onHeapMemoryUsed" : 0,
"offHeapMemoryUsed" : 0,
"maxOnHeapMemory" : 384093388,
"maxOffHeapMemory" : 524288000
"memoryMetrics": {
"usedOnHeapStorageMemory": 0,
"usedOffHeapStorageMemory": 0,
"totalOnHeapStorageMemory": 384093388,
"totalOffHeapStorageMemory": 524288000
}
} ]
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment