From e20d9b15655ffaf1c220fbad1a2aa1c2d45137ba Mon Sep 17 00:00:00 2001
From: Parag Chaudhari <paragpc@amazon.com>
Date: Fri, 20 Jan 2017 10:49:05 -0600
Subject: [PATCH] [SPARK-19069][CORE] Expose task 'status' and 'duration' in
spark history server REST API.
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## What changes were proposed in this pull request?
Although Spark history server UI shows task ‘status’ and ‘duration’ fields, it does not expose these fields in the REST API response. For the Spark history server API users, it is not possible to determine task status and duration. Spark history server has access to task status and duration from event log, but it is not exposing these in API. This patch is proposed to expose task ‘status’ and ‘duration’ fields in Spark history server REST API.
## How was this patch tested?
Modified existing test cases in org.apache.spark.deploy.history.HistoryServerSuite.
Author: Parag Chaudhari <paragpc@amazon.com>
Closes #16473 from paragpc/expose_task_status.
---
.../status/api/v1/AllStagesResource.scala | 2 +
.../org/apache/spark/status/api/v1/api.scala | 2 +
.../org/apache/spark/ui/jobs/StagePage.scala | 6 +-
.../org/apache/spark/ui/jobs/UIData.scala | 8 ++
.../one_stage_attempt_json_expectation.json | 16 +++
.../one_stage_json_expectation.json | 16 +++
.../stage_task_list_expectation.json | 40 +++++++
...multi_attempt_app_json_1__expectation.json | 16 +++
...multi_attempt_app_json_2__expectation.json | 16 +++
...k_list_w__offset___length_expectation.json | 100 ++++++++++++++++++
...stage_task_list_w__sortBy_expectation.json | 40 +++++++
...tBy_short_names___runtime_expectation.json | 40 +++++++
...rtBy_short_names__runtime_expectation.json | 40 +++++++
...age_with_accumulable_json_expectation.json | 16 +++
project/MimaExcludes.scala | 7 +-
15 files changed, 360 insertions(+), 5 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
index acb7c23079..1818935392 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
@@ -142,8 +142,10 @@ private[v1] object AllStagesResource {
index = uiData.taskInfo.index,
attempt = uiData.taskInfo.attemptNumber,
launchTime = new Date(uiData.taskInfo.launchTime),
+ duration = uiData.taskDuration,
executorId = uiData.taskInfo.executorId,
host = uiData.taskInfo.host,
+ status = uiData.taskInfo.status,
taskLocality = uiData.taskInfo.taskLocality.toString(),
speculative = uiData.taskInfo.speculative,
accumulatorUpdates = uiData.taskInfo.accumulables.map { convertAccumulableInfo },
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 7d035b11fa..c509398db1 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -158,8 +158,10 @@ class TaskData private[spark](
val index: Int,
val attempt: Int,
val launchTime: Date,
+ val duration: Option[Long] = None,
val executorId: String,
val host: String,
+ val status: String,
val taskLocality: String,
val speculative: Boolean,
val accumulatorUpdates: Seq[AccumulableInfo],
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
index 412ddfa9fa..ff17775008 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/StagePage.scala
@@ -887,10 +887,8 @@ private[ui] class TaskDataSource(
private def taskRow(taskData: TaskUIData): TaskTableRowData = {
val info = taskData.taskInfo
val metrics = taskData.metrics
- val duration = if (info.status == "RUNNING") info.timeRunning(currentTime)
- else metrics.map(_.executorRunTime).getOrElse(1L)
- val formatDuration = if (info.status == "RUNNING") UIUtils.formatDuration(duration)
- else metrics.map(m => UIUtils.formatDuration(m.executorRunTime)).getOrElse("")
+ val duration = taskData.taskDuration.getOrElse(1L)
+ val formatDuration = taskData.taskDuration.map(d => UIUtils.formatDuration(d)).getOrElse("")
val schedulerDelay = metrics.map(getSchedulerDelay(info, _, currentTime)).getOrElse(0L)
val gcTime = metrics.map(_.jvmGCTime).getOrElse(0L)
val taskDeserializationTime = metrics.map(_.executorDeserializeTime).getOrElse(0L)
diff --git a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
index 371dad966d..201e6197ba 100644
--- a/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
+++ b/core/src/main/scala/org/apache/spark/ui/jobs/UIData.scala
@@ -129,6 +129,14 @@ private[spark] object UIData {
def updateTaskMetrics(metrics: Option[TaskMetrics]): Unit = {
_metrics = TaskUIData.toTaskMetricsUIData(metrics)
}
+
+ def taskDuration: Option[Long] = {
+ if (taskInfo.status == "RUNNING") {
+ Some(_taskInfo.timeRunning(System.currentTimeMillis))
+ } else {
+ _metrics.map(_.executorRunTime)
+ }
+ }
}
object TaskUIData {
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index cdebb5f5b9..c2f450ba87 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -30,8 +30,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.829GMT",
+ "duration" : 435,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -73,8 +75,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
+ "duration" : 436,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -116,8 +120,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -159,8 +165,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -202,8 +210,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.831GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -245,8 +255,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.831GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -288,8 +300,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.832GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -331,8 +345,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.833GMT",
+ "duration" : 435,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 85dafcffc8..506859ae54 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -30,8 +30,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.829GMT",
+ "duration" : 435,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -73,8 +75,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
+ "duration" : 436,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -116,8 +120,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -159,8 +165,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.830GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -202,8 +210,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.831GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -245,8 +255,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.831GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -288,8 +300,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.832GMT",
+ "duration" : 434,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -331,8 +345,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-02-03T16:43:05.833GMT",
+ "duration" : 435,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
index e0661c4641..f4cec68fbf 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@@ -3,8 +3,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.494GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -45,8 +47,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.502GMT",
+ "duration" : 350,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -87,8 +91,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.503GMT",
+ "duration" : 348,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -129,8 +135,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -171,8 +179,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -213,8 +223,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.505GMT",
+ "duration" : 350,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -255,8 +267,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.505GMT",
+ "duration" : 351,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -297,8 +311,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.506GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -339,8 +355,10 @@
"index" : 8,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.914GMT",
+ "duration" : 80,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -381,8 +399,10 @@
"index" : 9,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.915GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -423,8 +443,10 @@
"index" : 10,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.916GMT",
+ "duration" : 73,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -465,8 +487,10 @@
"index" : 11,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.918GMT",
+ "duration" : 75,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -507,8 +531,10 @@
"index" : 12,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.923GMT",
+ "duration" : 77,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -549,8 +575,10 @@
"index" : 13,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.924GMT",
+ "duration" : 76,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -591,8 +619,10 @@
"index" : 14,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.925GMT",
+ "duration" : 83,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -633,8 +663,10 @@
"index" : 15,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.928GMT",
+ "duration" : 76,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -675,8 +707,10 @@
"index" : 16,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.001GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -717,8 +751,10 @@
"index" : 17,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.005GMT",
+ "duration" : 91,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -759,8 +795,10 @@
"index" : 18,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.010GMT",
+ "duration" : 92,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -801,8 +839,10 @@
"index" : 19,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.012GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
index 8492f19ab7..496a21c328 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@@ -3,8 +3,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.515GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -50,8 +52,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.521GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -97,8 +101,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -144,8 +150,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -191,8 +199,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -238,8 +248,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.523GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -285,8 +297,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.523GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -332,8 +346,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.524GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
index 4de4c501a4..4328dc753c 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@@ -3,8 +3,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.515GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -50,8 +52,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.521GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -97,8 +101,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -144,8 +150,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -191,8 +199,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -238,8 +248,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.523GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -285,8 +297,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.523GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -332,8 +346,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-03-17T23:12:16.524GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
index d2eceeb3f9..8c571430f3 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
@@ -3,8 +3,10 @@
"index" : 10,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.916GMT",
+ "duration" : 73,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -45,8 +47,10 @@
"index" : 11,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.918GMT",
+ "duration" : 75,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -87,8 +91,10 @@
"index" : 12,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.923GMT",
+ "duration" : 77,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -129,8 +135,10 @@
"index" : 13,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.924GMT",
+ "duration" : 76,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -171,8 +179,10 @@
"index" : 14,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.925GMT",
+ "duration" : 83,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -213,8 +223,10 @@
"index" : 15,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.928GMT",
+ "duration" : 76,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -255,8 +267,10 @@
"index" : 16,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.001GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -297,8 +311,10 @@
"index" : 17,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.005GMT",
+ "duration" : 91,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -339,8 +355,10 @@
"index" : 18,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.010GMT",
+ "duration" : 92,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -381,8 +399,10 @@
"index" : 19,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.012GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -423,8 +443,10 @@
"index" : 20,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.014GMT",
+ "duration" : 83,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -465,8 +487,10 @@
"index" : 21,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.015GMT",
+ "duration" : 88,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -507,8 +531,10 @@
"index" : 22,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.018GMT",
+ "duration" : 93,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -549,8 +575,10 @@
"index" : 23,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.031GMT",
+ "duration" : 65,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -591,8 +619,10 @@
"index" : 24,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.098GMT",
+ "duration" : 43,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -633,8 +663,10 @@
"index" : 25,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.103GMT",
+ "duration" : 49,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -675,8 +707,10 @@
"index" : 26,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.105GMT",
+ "duration" : 38,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -717,8 +751,10 @@
"index" : 27,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.110GMT",
+ "duration" : 32,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -759,8 +795,10 @@
"index" : 28,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.113GMT",
+ "duration" : 29,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -801,8 +839,10 @@
"index" : 29,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.114GMT",
+ "duration" : 39,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -843,8 +883,10 @@
"index" : 30,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.118GMT",
+ "duration" : 34,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -885,8 +927,10 @@
"index" : 31,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.127GMT",
+ "duration" : 24,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -927,8 +971,10 @@
"index" : 32,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.148GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -969,8 +1015,10 @@
"index" : 33,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.149GMT",
+ "duration" : 43,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1011,8 +1059,10 @@
"index" : 34,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.156GMT",
+ "duration" : 27,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1053,8 +1103,10 @@
"index" : 35,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.161GMT",
+ "duration" : 35,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1095,8 +1147,10 @@
"index" : 36,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.164GMT",
+ "duration" : 29,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1137,8 +1191,10 @@
"index" : 37,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.165GMT",
+ "duration" : 32,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1179,8 +1235,10 @@
"index" : 38,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.166GMT",
+ "duration" : 31,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1221,8 +1279,10 @@
"index" : 39,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.180GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1263,8 +1323,10 @@
"index" : 40,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.197GMT",
+ "duration" : 14,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1305,8 +1367,10 @@
"index" : 41,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.200GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1347,8 +1411,10 @@
"index" : 42,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.203GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1389,8 +1455,10 @@
"index" : 43,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.204GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1431,8 +1499,10 @@
"index" : 44,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.205GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1473,8 +1543,10 @@
"index" : 45,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.206GMT",
+ "duration" : 19,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1515,8 +1587,10 @@
"index" : 46,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.210GMT",
+ "duration" : 31,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1557,8 +1631,10 @@
"index" : 47,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.212GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1599,8 +1675,10 @@
"index" : 48,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.220GMT",
+ "duration" : 24,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1641,8 +1719,10 @@
"index" : 49,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.223GMT",
+ "duration" : 23,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1683,8 +1763,10 @@
"index" : 50,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.240GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1725,8 +1807,10 @@
"index" : 51,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.242GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1767,8 +1851,10 @@
"index" : 52,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.243GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1809,8 +1895,10 @@
"index" : 53,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.244GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1851,8 +1939,10 @@
"index" : 54,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.244GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1893,8 +1983,10 @@
"index" : 55,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.246GMT",
+ "duration" : 21,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1935,8 +2027,10 @@
"index" : 56,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.249GMT",
+ "duration" : 20,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -1977,8 +2071,10 @@
"index" : 57,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.257GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -2019,8 +2115,10 @@
"index" : 58,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.263GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -2061,8 +2159,10 @@
"index" : 59,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.265GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index f42c3a4ee5..0bd614bdc7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -3,8 +3,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.505GMT",
+ "duration" : 351,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -45,8 +47,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.502GMT",
+ "duration" : 350,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -87,8 +91,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.505GMT",
+ "duration" : 350,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -129,8 +135,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.494GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -171,8 +179,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -213,8 +223,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -255,8 +267,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.506GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -297,8 +311,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.503GMT",
+ "duration" : 348,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -339,8 +355,10 @@
"index" : 22,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.018GMT",
+ "duration" : 93,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -381,8 +399,10 @@
"index" : 18,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.010GMT",
+ "duration" : 92,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -423,8 +443,10 @@
"index" : 17,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.005GMT",
+ "duration" : 91,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -465,8 +487,10 @@
"index" : 21,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.015GMT",
+ "duration" : 88,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -507,8 +531,10 @@
"index" : 9,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.915GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -549,8 +575,10 @@
"index" : 16,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.001GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -591,8 +619,10 @@
"index" : 19,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.012GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -633,8 +663,10 @@
"index" : 14,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.925GMT",
+ "duration" : 83,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -675,8 +707,10 @@
"index" : 20,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.014GMT",
+ "duration" : 83,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -717,8 +751,10 @@
"index" : 8,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.914GMT",
+ "duration" : 80,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -759,8 +795,10 @@
"index" : 12,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.923GMT",
+ "duration" : 77,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -801,8 +839,10 @@
"index" : 13,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.924GMT",
+ "duration" : 76,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index f42c3a4ee5..0bd614bdc7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -3,8 +3,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.505GMT",
+ "duration" : 351,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -45,8 +47,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.502GMT",
+ "duration" : 350,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -87,8 +91,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.505GMT",
+ "duration" : 350,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -129,8 +135,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.494GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -171,8 +179,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -213,8 +223,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.504GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -255,8 +267,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.506GMT",
+ "duration" : 349,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -297,8 +311,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.503GMT",
+ "duration" : 348,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -339,8 +355,10 @@
"index" : 22,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.018GMT",
+ "duration" : 93,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -381,8 +399,10 @@
"index" : 18,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.010GMT",
+ "duration" : 92,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -423,8 +443,10 @@
"index" : 17,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.005GMT",
+ "duration" : 91,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -465,8 +487,10 @@
"index" : 21,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.015GMT",
+ "duration" : 88,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -507,8 +531,10 @@
"index" : 9,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.915GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -549,8 +575,10 @@
"index" : 16,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.001GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -591,8 +619,10 @@
"index" : 19,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.012GMT",
+ "duration" : 84,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -633,8 +663,10 @@
"index" : 14,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.925GMT",
+ "duration" : 83,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -675,8 +707,10 @@
"index" : 20,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.014GMT",
+ "duration" : 83,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -717,8 +751,10 @@
"index" : 8,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.914GMT",
+ "duration" : 80,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -759,8 +795,10 @@
"index" : 12,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.923GMT",
+ "duration" : 77,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -801,8 +839,10 @@
"index" : 13,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:06.924GMT",
+ "duration" : 76,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index db60ccccbf..b58f1a51ba 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -3,8 +3,10 @@
"index" : 40,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.197GMT",
+ "duration" : 14,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -45,8 +47,10 @@
"index" : 41,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.200GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -87,8 +91,10 @@
"index" : 43,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.204GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -129,8 +135,10 @@
"index" : 57,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.257GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -171,8 +179,10 @@
"index" : 58,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.263GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -213,8 +223,10 @@
"index" : 68,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.306GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -255,8 +267,10 @@
"index" : 86,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.374GMT",
+ "duration" : 16,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -297,8 +311,10 @@
"index" : 32,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.148GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -339,8 +355,10 @@
"index" : 39,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.180GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -381,8 +399,10 @@
"index" : 42,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.203GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -423,8 +443,10 @@
"index" : 51,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.242GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -465,8 +487,10 @@
"index" : 59,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.265GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -507,8 +531,10 @@
"index" : 63,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.276GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -549,8 +575,10 @@
"index" : 87,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.374GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -591,8 +619,10 @@
"index" : 90,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.385GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -633,8 +663,10 @@
"index" : 99,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.426GMT",
+ "duration" : 17,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -675,8 +707,10 @@
"index" : 44,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.205GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -717,8 +751,10 @@
"index" : 47,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.212GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -759,8 +795,10 @@
"index" : 50,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.240GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
@@ -801,8 +839,10 @@
"index" : 52,
"attempt" : 0,
"launchTime" : "2015-05-06T13:03:07.243GMT",
+ "duration" : 18,
"executorId" : "driver",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ ],
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index c2ece6f4d0..a449926ee7 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -34,8 +34,10 @@
"index" : 0,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.515GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -82,8 +84,10 @@
"index" : 1,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.521GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -130,8 +134,10 @@
"index" : 2,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -178,8 +184,10 @@
"index" : 3,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -226,8 +234,10 @@
"index" : 4,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.522GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -274,8 +284,10 @@
"index" : 5,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.523GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -322,8 +334,10 @@
"index" : 6,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.523GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
@@ -370,8 +384,10 @@
"index" : 7,
"attempt" : 0,
"launchTime" : "2015-03-16T19:25:36.524GMT",
+ "duration" : 15,
"executorId" : "<driver>",
"host" : "localhost",
+ "status" : "SUCCESS",
"taskLocality" : "PROCESS_LOCAL",
"speculative" : false,
"accumulatorUpdates" : [ {
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index bf628210a1..7e6e143523 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -49,7 +49,12 @@ object MimaExcludes {
ProblemFilters.exclude[ReversedMissingMethodProblem]("org.apache.spark.sql.catalog.Catalog.createTable"),
// [SPARK-14272][ML] Add logLikelihood in GaussianMixtureSummary
- ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.clustering.GaussianMixtureSummary.this")
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.ml.clustering.GaussianMixtureSummary.this"),
+
+ // [SPARK-19069] [CORE] Expose task 'status' and 'duration' in spark history server REST API.
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskData.this"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskData.<init>$default$10"),
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.TaskData.<init>$default$11")
)
// Exclude rules for 2.1.x
--
GitLab