Skip to content
Snippets Groups Projects
Commit 61c45876 authored by Alex Bozarth's avatar Alex Bozarth Committed by Tom Graves
Browse files

[SPARK-12716][WEB UI] Add a TOTALS row to the Executors Web UI

Added a Totals table to the top of the page to display the totals of each applicable column in the executors table.

Old Description:
~~Created a TOTALS row containing the totals of each column in the executors UI. By default the TOTALS row appears at the top of the table. When a column is sorted the TOTALS row will always sort to either the top or bottom of the table.~~

Author: Alex Bozarth <ajbozart@us.ibm.com>

Closes #10668 from ajbozarth/spark12716.
parent 0bb73554
No related branches found
No related tags found
No related merge requests found
...@@ -62,9 +62,6 @@ private[ui] class ExecutorsPage( ...@@ -62,9 +62,6 @@ private[ui] class ExecutorsPage(
} }
(_storageStatusList, _execInfo) (_storageStatusList, _execInfo)
} }
val maxMem = storageStatusList.map(_.maxMem).sum
val memUsed = storageStatusList.map(_.memUsed).sum
val diskUsed = storageStatusList.map(_.diskUsed).sum
val execInfoSorted = execInfo.sortBy(_.id) val execInfoSorted = execInfo.sortBy(_.id)
val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty val logsExist = execInfo.filter(_.executorLogs.nonEmpty).nonEmpty
...@@ -100,18 +97,15 @@ private[ui] class ExecutorsPage( ...@@ -100,18 +97,15 @@ private[ui] class ExecutorsPage(
</table> </table>
val content = val content =
<div class="row-fluid"> <div class="row">
<div class="span12"> <div class="span12">
<ul class="unstyled"> <h4>Totals for {execInfo.size} Executors</h4>
<li><strong>Memory:</strong> {execSummary(execInfo)}
{Utils.bytesToString(memUsed)} Used
({Utils.bytesToString(maxMem)} Total) </li>
<li><strong>Disk:</strong> {Utils.bytesToString(diskUsed)} Used </li>
</ul>
</div> </div>
</div> </div>
<div class = "row"> <div class = "row">
<div class="span12"> <div class="span12">
<h4>Active Executors</h4>
{execTable} {execTable}
</div> </div>
</div>; </div>;
...@@ -179,6 +173,66 @@ private[ui] class ExecutorsPage( ...@@ -179,6 +173,66 @@ private[ui] class ExecutorsPage(
</tr> </tr>
} }
private def execSummary(execInfo: Seq[ExecutorSummary]): Seq[Node] = {
val maximumMemory = execInfo.map(_.maxMemory).sum
val memoryUsed = execInfo.map(_.memoryUsed).sum
val diskUsed = execInfo.map(_.diskUsed).sum
val totalDuration = execInfo.map(_.totalDuration).sum
val totalInputBytes = execInfo.map(_.totalInputBytes).sum
val totalShuffleRead = execInfo.map(_.totalShuffleRead).sum
val totalShuffleWrite = execInfo.map(_.totalShuffleWrite).sum
val sumContent =
<tr>
<td>{execInfo.map(_.rddBlocks).sum}</td>
<td sorttable_customkey={memoryUsed.toString}>
{Utils.bytesToString(memoryUsed)} /
{Utils.bytesToString(maximumMemory)}
</td>
<td sorttable_customkey={diskUsed.toString}>
{Utils.bytesToString(diskUsed)}
</td>
<td>{execInfo.map(_.activeTasks).sum}</td>
<td>{execInfo.map(_.failedTasks).sum}</td>
<td>{execInfo.map(_.completedTasks).sum}</td>
<td>{execInfo.map(_.totalTasks).sum}</td>
<td sorttable_customkey={totalDuration.toString}>
{Utils.msDurationToString(totalDuration)}
</td>
<td sorttable_customkey={totalInputBytes.toString}>
{Utils.bytesToString(totalInputBytes)}
</td>
<td sorttable_customkey={totalShuffleRead.toString}>
{Utils.bytesToString(totalShuffleRead)}
</td>
<td sorttable_customkey={totalShuffleWrite.toString}>
{Utils.bytesToString(totalShuffleWrite)}
</td>
</tr>;
<table class={UIUtils.TABLE_CLASS_STRIPED}>
<thead>
<th>RDD Blocks</th>
<th><span data-toggle="tooltip" title={ToolTips.STORAGE_MEMORY}>Storage Memory</span></th>
<th>Disk Used</th>
<th>Active Tasks</th>
<th>Failed Tasks</th>
<th>Complete Tasks</th>
<th>Total Tasks</th>
<th>Task Time</th>
<th><span data-toggle="tooltip" title={ToolTips.INPUT}>Input</span></th>
<th><span data-toggle="tooltip" title={ToolTips.SHUFFLE_READ}>Shuffle Read</span></th>
<th>
<span data-toggle="tooltip" data-placement="left" title={ToolTips.SHUFFLE_WRITE}>
Shuffle Write
</span>
</th>
</thead>
<tbody>
{sumContent}
</tbody>
</table>
}
} }
private[spark] object ExecutorsPage { private[spark] object ExecutorsPage {
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment