From abc6c42c2d76d6aa4a8cac605cb4214e8f8f3328 Mon Sep 17 00:00:00 2001
From: Alex Bozarth <ajbozart@us.ibm.com>
Date: Fri, 1 Apr 2016 16:18:09 -0700
Subject: [PATCH] [SPARK-13241][WEB UI] Added long values for dates in
ApplicationAttemptInfo API
## What changes were proposed in this pull request?
Adding long values for each Date in the ApplicationAttemptInfo API for easier use in code
## How was the this patch tested?
Tested with dev/run-tests
Author: Alex Bozarth <ajbozart@us.ibm.com>
Closes #11326 from ajbozarth/spark13241.
---
.../org/apache/spark/status/api/v1/api.scala | 6 ++++-
.../application_list_json_expectation.json | 24 +++++++++++++++++++
.../completed_app_list_json_expectation.json | 24 +++++++++++++++++++
.../maxDate2_app_list_json_expectation.json | 3 +++
.../maxDate_app_list_json_expectation.json | 6 +++++
.../minDate_app_list_json_expectation.json | 18 ++++++++++++++
.../one_app_json_expectation.json | 3 +++
...ne_app_multi_attempt_json_expectation.json | 6 +++++
.../deploy/history/HistoryServerSuite.scala | 4 +++-
9 files changed, 92 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index d43868bbcb..ebbbf48148 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -38,7 +38,11 @@ class ApplicationAttemptInfo private[spark](
val lastUpdated: Date,
val duration: Long,
val sparkUser: String,
- val completed: Boolean = false)
+ val completed: Boolean = false) {
+ def getStartTimeEpoch: Long = startTime.getTime
+ def getEndTimeEpoch: Long = endTime.getTime
+ def getLastUpdatedEpoch: Long = lastUpdated.getTime
+}
class ExecutorStageSummary private[spark](
val taskTime : Long,
diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
index 5bbb4ceb97..1a13233133 100644
--- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@@ -2,6 +2,9 @@
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:11.398GMT",
"lastUpdated" : "",
@@ -14,6 +17,9 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:00.950GMT",
"lastUpdated" : "",
@@ -22,6 +28,9 @@
"completed" : true
}, {
"attemptId" : "1",
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.880GMT",
"endTime" : "2015-05-06T13:03:00.890GMT",
"lastUpdated" : "",
@@ -34,6 +43,9 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
@@ -42,6 +54,9 @@
"completed" : true
}, {
"attemptId" : "1",
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
@@ -53,6 +68,9 @@
"id" : "local-1425081759269",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1425081758277,
+ "endTimeEpoch" : 1425081766912,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-28T00:02:38.277GMT",
"endTime" : "2015-02-28T00:02:46.912GMT",
"lastUpdated" : "",
@@ -64,6 +82,9 @@
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
@@ -75,6 +96,9 @@
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
index 5bbb4ceb97..1a13233133 100644
--- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@@ -2,6 +2,9 @@
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:11.398GMT",
"lastUpdated" : "",
@@ -14,6 +17,9 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:00.950GMT",
"lastUpdated" : "",
@@ -22,6 +28,9 @@
"completed" : true
}, {
"attemptId" : "1",
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.880GMT",
"endTime" : "2015-05-06T13:03:00.890GMT",
"lastUpdated" : "",
@@ -34,6 +43,9 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
@@ -42,6 +54,9 @@
"completed" : true
}, {
"attemptId" : "1",
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
@@ -53,6 +68,9 @@
"id" : "local-1425081759269",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1425081758277,
+ "endTimeEpoch" : 1425081766912,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-28T00:02:38.277GMT",
"endTime" : "2015-02-28T00:02:46.912GMT",
"lastUpdated" : "",
@@ -64,6 +82,9 @@
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
@@ -75,6 +96,9 @@
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
index 3f80a529a0..eacf04b901 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
@@ -2,6 +2,9 @@
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
index 508bdc17ef..adad25bf17 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
@@ -2,6 +2,9 @@
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
@@ -13,6 +16,9 @@
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
index 5dca7d73de..a658909088 100644
--- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@@ -2,6 +2,9 @@
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:11.398GMT",
"lastUpdated" : "",
@@ -14,6 +17,9 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:00.950GMT",
"lastUpdated" : "",
@@ -22,6 +28,9 @@
"completed" : true
}, {
"attemptId" : "1",
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.880GMT",
"endTime" : "2015-05-06T13:03:00.890GMT",
"lastUpdated" : "",
@@ -34,6 +43,9 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
@@ -42,6 +54,9 @@
"completed" : true
}, {
"attemptId" : "1",
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
@@ -54,6 +69,9 @@
"name": "Spark shell",
"attempts": [
{
+ "startTimeEpoch" : 1425081758277,
+ "endTimeEpoch" : 1425081766912,
+ "lastUpdatedEpoch" : 0,
"startTime": "2015-02-28T00:02:38.277GMT",
"endTime": "2015-02-28T00:02:46.912GMT",
"lastUpdated" : "",
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
index cca32c7910..0217facad9 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
@@ -2,6 +2,9 @@
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
index 1ea1779e83..b20a26648e 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
@@ -3,6 +3,9 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
@@ -11,6 +14,9 @@
"completed" : true
}, {
"attemptId" : "1",
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 79e4efb1a8..2a013aca7b 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -162,7 +162,9 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
val json = if (jsonOrg.indexOf("lastUpdated") >= 0) {
val subStrings = jsonOrg.split(",")
for (i <- subStrings.indices) {
- if (subStrings(i).indexOf("lastUpdated") >= 0) {
+ if (subStrings(i).indexOf("lastUpdatedEpoch") >= 0) {
+ subStrings(i) = subStrings(i).replaceAll("(\\d+)", "0")
+ } else if (subStrings(i).indexOf("lastUpdated") >= 0) {
subStrings(i) = "\"lastUpdated\":\"\""
}
}
--
GitLab