Skip to content
Snippets Groups Projects
Commit 182f9bae authored by Patrick Wendell's avatar Patrick Wendell
Browse files

Merge pull request #227 from pwendell/master

Fix small bug in web UI and minor clean-up.

There was a bug where sorting order didn't work correctly for write time metrics.

I also cleaned up some earlier code that fixed the same issue for read and
write bytes.
parents b9e7609f 380b90b9
No related branches found
No related tags found
No related merge requests found
...@@ -152,21 +152,18 @@ private[spark] class StagePage(parent: JobProgressUI) { ...@@ -152,21 +152,18 @@ private[spark] class StagePage(parent: JobProgressUI) {
else metrics.map(m => parent.formatDuration(m.executorRunTime)).getOrElse("") else metrics.map(m => parent.formatDuration(m.executorRunTime)).getOrElse("")
val gcTime = metrics.map(m => m.jvmGCTime).getOrElse(0L) val gcTime = metrics.map(m => m.jvmGCTime).getOrElse(0L)
var shuffleReadSortable: String = "" val maybeShuffleRead = metrics.flatMap{m => m.shuffleReadMetrics}.map{s => s.remoteBytesRead}
var shuffleReadReadable: String = "" val shuffleReadSortable = maybeShuffleRead.map(_.toString).getOrElse("")
if (shuffleRead) { val shuffleReadReadable = maybeShuffleRead.map{Utils.bytesToString(_)}.getOrElse("")
shuffleReadSortable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s => s.remoteBytesRead}.toString()
shuffleReadReadable = metrics.flatMap{m => m.shuffleReadMetrics}.map{s =>
Utils.bytesToString(s.remoteBytesRead)}.getOrElse("")
}
var shuffleWriteSortable: String = "" val maybeShuffleWrite = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => s.shuffleBytesWritten}
var shuffleWriteReadable: String = "" val shuffleWriteSortable = maybeShuffleWrite.map(_.toString).getOrElse("")
if (shuffleWrite) { val shuffleWriteReadable = maybeShuffleWrite.map{Utils.bytesToString(_)}.getOrElse("")
shuffleWriteSortable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => s.shuffleBytesWritten}.toString()
shuffleWriteReadable = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => val maybeWriteTime = metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => s.shuffleWriteTime}
Utils.bytesToString(s.shuffleBytesWritten)}.getOrElse("") val writeTimeSortable = maybeWriteTime.map(_.toString).getOrElse("")
} val writeTimeReadable = maybeWriteTime.map{ t => t / (1000 * 1000)}.map{ ms =>
if (ms == 0) "" else parent.formatDuration(ms)}.getOrElse("")
<tr> <tr>
<td>{info.index}</td> <td>{info.index}</td>
...@@ -187,8 +184,8 @@ private[spark] class StagePage(parent: JobProgressUI) { ...@@ -187,8 +184,8 @@ private[spark] class StagePage(parent: JobProgressUI) {
</td> </td>
}} }}
{if (shuffleWrite) { {if (shuffleWrite) {
<td>{metrics.flatMap{m => m.shuffleWriteMetrics}.map{s => <td sorttable_customkey={writeTimeSortable}>
parent.formatDuration(s.shuffleWriteTime / (1000 * 1000))}.getOrElse("")} {writeTimeReadable}
</td> </td>
<td sorttable_customkey={shuffleWriteSortable}> <td sorttable_customkey={shuffleWriteSortable}>
{shuffleWriteReadable} {shuffleWriteReadable}
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment