From 630bd5fd80193ab6dc6ad0e7bcc13ee0dadabd38 Mon Sep 17 00:00:00 2001
From: zsxwing <zsxwing@gmail.com>
Date: Tue, 30 Jun 2015 00:46:55 +0900
Subject: [PATCH] [SPARK-8702] [WEBUI] Avoid massive concating strings in
Javascript
When there are massive tasks, such as `sc.parallelize(1 to 100000, 10000).count()`, the generated JS codes have a lot of string concatenations in the stage page, nearly 40 string concatenations for one task.
We can generate the whole string for a task instead of execution string concatenations in the browser.
Before this patch, the load time of the page is about 21 seconds.
After this patch, it reduces to about 17 seconds.
One disadvantage is that the generated JS codes become hard to read.
Author: zsxwing <zsxwing@gmail.com>
Closes #7082 from zsxwing/js-string and squashes the following commits:
b29231d [zsxwing] Avoid massive concating strings in Javascript
---
.../org/apache/spark/ui/jobs/StagePage.scala | 88 +++++++++----------
1 file changed, 44 insertions(+), 44 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index b83a49f79c..e96bf49d0d 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -572,55 +572,55 @@ private[ui] class StagePage(parent: StagesTab) extends WebUIPage("stage") {
val attempt = taskInfo.attempt
val timelineObject =
s"""
- {
- 'className': 'task task-assignment-timeline-object',
- 'group': '$executorId',
- 'content': '<div class="task-assignment-timeline-content"' +
- 'data-toggle="tooltip" data-placement="top"' +
- 'data-html="true" data-container="body"' +
- 'data-title="${s"Task " + index + " (attempt " + attempt + ")"}<br>' +
- 'Status: ${taskInfo.status}<br>' +
- 'Launch Time: ${UIUtils.formatDate(new Date(launchTime))}' +
- '${
+ |{
+ |'className': 'task task-assignment-timeline-object',
+ |'group': '$executorId',
+ |'content': '<div class="task-assignment-timeline-content"
+ |data-toggle="tooltip" data-placement="top"
+ |data-html="true" data-container="body"
+ |data-title="${s"Task " + index + " (attempt " + attempt + ")"}<br>
+ |Status: ${taskInfo.status}<br>
+ |Launch Time: ${UIUtils.formatDate(new Date(launchTime))}
+ |${
if (!taskInfo.running) {
s"""<br>Finish Time: ${UIUtils.formatDate(new Date(finishTime))}"""
} else {
""
}
- }' +
- '<br>Scheduler Delay: $schedulerDelay ms' +
- '<br>Task Deserialization Time: ${UIUtils.formatDuration(deserializationTime)}' +
- '<br>Shuffle Read Time: ${UIUtils.formatDuration(shuffleReadTime)}' +
- '<br>Executor Computing Time: ${UIUtils.formatDuration(executorComputingTime)}' +
- '<br>Shuffle Write Time: ${UIUtils.formatDuration(shuffleWriteTime)}' +
- '<br>Result Serialization Time: ${UIUtils.formatDuration(serializationTime)}' +
- '<br>Getting Result Time: ${UIUtils.formatDuration(gettingResultTime)}">' +
- '<svg class="task-assignment-timeline-duration-bar">' +
- '<rect class="scheduler-delay-proportion" ' +
- 'x="$schedulerDelayProportionPos%" y="0px" height="26px"' +
- 'width="$schedulerDelayProportion%""></rect>' +
- '<rect class="deserialization-time-proportion" '+
- 'x="$deserializationTimeProportionPos%" y="0px" height="26px"' +
- 'width="$deserializationTimeProportion%"></rect>' +
- '<rect class="shuffle-read-time-proportion" ' +
- 'x="$shuffleReadTimeProportionPos%" y="0px" height="26px"' +
- 'width="$shuffleReadTimeProportion%"></rect>' +
- '<rect class="executor-runtime-proportion" ' +
- 'x="$executorRuntimeProportionPos%" y="0px" height="26px"' +
- 'width="$executorComputingTimeProportion%"></rect>' +
- '<rect class="shuffle-write-time-proportion" ' +
- 'x="$shuffleWriteTimeProportionPos%" y="0px" height="26px"' +
- 'width="$shuffleWriteTimeProportion%"></rect>' +
- '<rect class="serialization-time-proportion" ' +
- 'x="$serializationTimeProportionPos%" y="0px" height="26px"' +
- 'width="$serializationTimeProportion%"></rect>' +
- '<rect class="getting-result-time-proportion" ' +
- 'x="$gettingResultTimeProportionPos%" y="0px" height="26px"' +
- 'width="$gettingResultTimeProportion%"></rect></svg>',
- 'start': new Date($launchTime),
- 'end': new Date($finishTime)
- }
- """
+ }
+ |<br>Scheduler Delay: $schedulerDelay ms
+ |<br>Task Deserialization Time: ${UIUtils.formatDuration(deserializationTime)}
+ |<br>Shuffle Read Time: ${UIUtils.formatDuration(shuffleReadTime)}
+ |<br>Executor Computing Time: ${UIUtils.formatDuration(executorComputingTime)}
+ |<br>Shuffle Write Time: ${UIUtils.formatDuration(shuffleWriteTime)}
+ |<br>Result Serialization Time: ${UIUtils.formatDuration(serializationTime)}
+ |<br>Getting Result Time: ${UIUtils.formatDuration(gettingResultTime)}">
+ |<svg class="task-assignment-timeline-duration-bar">
+ |<rect class="scheduler-delay-proportion"
+ |x="$schedulerDelayProportionPos%" y="0px" height="26px"
+ |width="$schedulerDelayProportion%""></rect>
+ |<rect class="deserialization-time-proportion"
+ |x="$deserializationTimeProportionPos%" y="0px" height="26px"
+ |width="$deserializationTimeProportion%"></rect>
+ |<rect class="shuffle-read-time-proportion"
+ |x="$shuffleReadTimeProportionPos%" y="0px" height="26px"
+ |width="$shuffleReadTimeProportion%"></rect>
+ |<rect class="executor-runtime-proportion"
+ |x="$executorRuntimeProportionPos%" y="0px" height="26px"
+ |width="$executorComputingTimeProportion%"></rect>
+ |<rect class="shuffle-write-time-proportion"
+ |x="$shuffleWriteTimeProportionPos%" y="0px" height="26px"
+ |width="$shuffleWriteTimeProportion%"></rect>
+ |<rect class="serialization-time-proportion"
+ |x="$serializationTimeProportionPos%" y="0px" height="26px"
+ |width="$serializationTimeProportion%"></rect>
+ |<rect class="getting-result-time-proportion"
+ |x="$gettingResultTimeProportionPos%" y="0px" height="26px"
+ |width="$gettingResultTimeProportion%"></rect></svg>',
+ |'start': new Date($launchTime),
+ |'end': new Date($finishTime)
+ |}
+ |""".stripMargin.replaceAll("\n", " ")
timelineObject
}.mkString("[", ",", "]")
--
GitLab