diff --git a/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala b/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala
index 2181bde9f05e1459a4bc834d249f0c5228c86866..0ec81d8d3588a0bfe40cc5871189d858fbe413ce 100644
--- a/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/InputMetrics.scala
@@ -58,11 +58,6 @@ class InputMetrics private (_bytesRead: Accumulator[Long], _recordsRead: Accumul
*/
def recordsRead: Long = _recordsRead.localValue
- /**
- * Returns true if this metrics has been updated before.
- */
- def isUpdated: Boolean = (bytesRead | recordsRead) != 0
-
private[spark] def incBytesRead(v: Long): Unit = _bytesRead.add(v)
private[spark] def incRecordsRead(v: Long): Unit = _recordsRead.add(v)
private[spark] def setBytesRead(v: Long): Unit = _bytesRead.setValue(v)
diff --git a/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala b/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala
index 7f20f6bf0d35d44a69df73982e796bb3f1037e92..5b36cc4739373b766795b3a44780b67066939d4e 100644
--- a/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/OutputMetrics.scala
@@ -57,11 +57,6 @@ class OutputMetrics private (_bytesWritten: Accumulator[Long], _recordsWritten:
*/
def recordsWritten: Long = _recordsWritten.localValue
- /**
- * Returns true if this metrics has been updated before.
- */
- def isUpdated: Boolean = (bytesWritten | recordsWritten) != 0
-
private[spark] def setBytesWritten(v: Long): Unit = _bytesWritten.setValue(v)
private[spark] def setRecordsWritten(v: Long): Unit = _recordsWritten.setValue(v)
}
diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala
index 9c78995ff35e75b46e85f7283795f79fb4cc4146..47cfb74b9eb7d301cd99be7e307bec4cd029df7a 100644
--- a/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ShuffleReadMetrics.scala
@@ -102,11 +102,6 @@ class ShuffleReadMetrics private (
*/
def totalBlocksFetched: Int = remoteBlocksFetched + localBlocksFetched
- /**
- * Returns true if this metrics has been updated before.
- */
- def isUpdated: Boolean = (totalBytesRead | totalBlocksFetched | recordsRead | fetchWaitTime) != 0
-
private[spark] def incRemoteBlocksFetched(v: Int): Unit = _remoteBlocksFetched.add(v)
private[spark] def incLocalBlocksFetched(v: Int): Unit = _localBlocksFetched.add(v)
private[spark] def incRemoteBytesRead(v: Long): Unit = _remoteBytesRead.add(v)
diff --git a/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala b/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala
index cf570e1f9d99f7c4c3a8c7d50bf0a568ce8b792a..704dee747eda107aad6a5870702e39eaf2d35877 100644
--- a/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala
+++ b/core/src/main/scala/org/apache/spark/executor/ShuffleWriteMetrics.scala
@@ -68,11 +68,6 @@ class ShuffleWriteMetrics private (
*/
def writeTime: Long = _writeTime.localValue
- /**
- * Returns true if this metrics has been updated before.
- */
- def isUpdated: Boolean = (writeTime | recordsWritten | bytesWritten) != 0
-
private[spark] def incBytesWritten(v: Long): Unit = _bytesWritten.add(v)
private[spark] def incRecordsWritten(v: Long): Unit = _recordsWritten.add(v)
private[spark] def incWriteTime(v: Long): Unit = _writeTime.add(v)
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
index 85452d6497a822f2dff8487b9b87ba6a55dec689..eddc36edc961151b0879da58bf6e3068d0380356 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/AllStagesResource.scala
@@ -167,47 +167,32 @@ private[v1] object AllStagesResource {
// to make it a little easier to deal w/ all of the nested options. Mostly it lets us just
// implement one "build" method, which just builds the quantiles for each field.
- val inputMetrics: Option[InputMetricDistributions] =
+ val inputMetrics: InputMetricDistributions =
new MetricHelper[InternalInputMetrics, InputMetricDistributions](rawMetrics, quantiles) {
- def getSubmetrics(raw: InternalTaskMetrics): Option[InternalInputMetrics] = {
- if (raw.inputMetrics.isUpdated) {
- Some(raw.inputMetrics)
- } else {
- None
- }
- }
+ def getSubmetrics(raw: InternalTaskMetrics): InternalInputMetrics = raw.inputMetrics
def build: InputMetricDistributions = new InputMetricDistributions(
bytesRead = submetricQuantiles(_.bytesRead),
recordsRead = submetricQuantiles(_.recordsRead)
)
- }.metricOption
+ }.build
- val outputMetrics: Option[OutputMetricDistributions] =
+ val outputMetrics: OutputMetricDistributions =
new MetricHelper[InternalOutputMetrics, OutputMetricDistributions](rawMetrics, quantiles) {
- def getSubmetrics(raw: InternalTaskMetrics): Option[InternalOutputMetrics] = {
- if (raw.outputMetrics.isUpdated) {
- Some(raw.outputMetrics)
- } else {
- None
- }
- }
+ def getSubmetrics(raw: InternalTaskMetrics): InternalOutputMetrics = raw.outputMetrics
+
def build: OutputMetricDistributions = new OutputMetricDistributions(
bytesWritten = submetricQuantiles(_.bytesWritten),
recordsWritten = submetricQuantiles(_.recordsWritten)
)
- }.metricOption
+ }.build
- val shuffleReadMetrics: Option[ShuffleReadMetricDistributions] =
+ val shuffleReadMetrics: ShuffleReadMetricDistributions =
new MetricHelper[InternalShuffleReadMetrics, ShuffleReadMetricDistributions](rawMetrics,
quantiles) {
- def getSubmetrics(raw: InternalTaskMetrics): Option[InternalShuffleReadMetrics] = {
- if (raw.shuffleReadMetrics.isUpdated) {
- Some(raw.shuffleReadMetrics)
- } else {
- None
- }
- }
+ def getSubmetrics(raw: InternalTaskMetrics): InternalShuffleReadMetrics =
+ raw.shuffleReadMetrics
+
def build: ShuffleReadMetricDistributions = new ShuffleReadMetricDistributions(
readBytes = submetricQuantiles(_.totalBytesRead),
readRecords = submetricQuantiles(_.recordsRead),
@@ -217,24 +202,20 @@ private[v1] object AllStagesResource {
totalBlocksFetched = submetricQuantiles(_.totalBlocksFetched),
fetchWaitTime = submetricQuantiles(_.fetchWaitTime)
)
- }.metricOption
+ }.build
- val shuffleWriteMetrics: Option[ShuffleWriteMetricDistributions] =
+ val shuffleWriteMetrics: ShuffleWriteMetricDistributions =
new MetricHelper[InternalShuffleWriteMetrics, ShuffleWriteMetricDistributions](rawMetrics,
quantiles) {
- def getSubmetrics(raw: InternalTaskMetrics): Option[InternalShuffleWriteMetrics] = {
- if (raw.shuffleWriteMetrics.isUpdated) {
- Some(raw.shuffleWriteMetrics)
- } else {
- None
- }
- }
+ def getSubmetrics(raw: InternalTaskMetrics): InternalShuffleWriteMetrics =
+ raw.shuffleWriteMetrics
+
def build: ShuffleWriteMetricDistributions = new ShuffleWriteMetricDistributions(
writeBytes = submetricQuantiles(_.bytesWritten),
writeRecords = submetricQuantiles(_.recordsWritten),
writeTime = submetricQuantiles(_.writeTime)
)
- }.metricOption
+ }.build
new TaskMetricDistributions(
quantiles = quantiles,
@@ -273,84 +254,55 @@ private[v1] object AllStagesResource {
)
}
- def convertInputMetrics(internal: InternalInputMetrics): Option[InputMetrics] = {
- if (internal.isUpdated) {
- Some(new InputMetrics(
- bytesRead = internal.bytesRead,
- recordsRead = internal.recordsRead
- ))
- } else {
- None
- }
+ def convertInputMetrics(internal: InternalInputMetrics): InputMetrics = {
+ new InputMetrics(
+ bytesRead = internal.bytesRead,
+ recordsRead = internal.recordsRead
+ )
}
- def convertOutputMetrics(internal: InternalOutputMetrics): Option[OutputMetrics] = {
- if (internal.isUpdated) {
- Some(new OutputMetrics(
- bytesWritten = internal.bytesWritten,
- recordsWritten = internal.recordsWritten
- ))
- } else {
- None
- }
+ def convertOutputMetrics(internal: InternalOutputMetrics): OutputMetrics = {
+ new OutputMetrics(
+ bytesWritten = internal.bytesWritten,
+ recordsWritten = internal.recordsWritten
+ )
}
- def convertShuffleReadMetrics(
- internal: InternalShuffleReadMetrics): Option[ShuffleReadMetrics] = {
- if (internal.isUpdated) {
- Some(new ShuffleReadMetrics(
- remoteBlocksFetched = internal.remoteBlocksFetched,
- localBlocksFetched = internal.localBlocksFetched,
- fetchWaitTime = internal.fetchWaitTime,
- remoteBytesRead = internal.remoteBytesRead,
- totalBlocksFetched = internal.totalBlocksFetched,
- recordsRead = internal.recordsRead
- ))
- } else {
- None
- }
+ def convertShuffleReadMetrics(internal: InternalShuffleReadMetrics): ShuffleReadMetrics = {
+ new ShuffleReadMetrics(
+ remoteBlocksFetched = internal.remoteBlocksFetched,
+ localBlocksFetched = internal.localBlocksFetched,
+ fetchWaitTime = internal.fetchWaitTime,
+ remoteBytesRead = internal.remoteBytesRead,
+ localBytesRead = internal.localBytesRead,
+ recordsRead = internal.recordsRead
+ )
}
- def convertShuffleWriteMetrics(
- internal: InternalShuffleWriteMetrics): Option[ShuffleWriteMetrics] = {
- if ((internal.bytesWritten | internal.writeTime | internal.recordsWritten) == 0) {
- None
- } else {
- Some(new ShuffleWriteMetrics(
- bytesWritten = internal.bytesWritten,
- writeTime = internal.writeTime,
- recordsWritten = internal.recordsWritten
- ))
- }
+ def convertShuffleWriteMetrics(internal: InternalShuffleWriteMetrics): ShuffleWriteMetrics = {
+ new ShuffleWriteMetrics(
+ bytesWritten = internal.bytesWritten,
+ writeTime = internal.writeTime,
+ recordsWritten = internal.recordsWritten
+ )
}
}
/**
- * Helper for getting distributions from nested metric types. Many of the metrics we want are
- * contained in options inside TaskMetrics (eg., ShuffleWriteMetrics). This makes it easy to handle
- * the options (returning None if the metrics are all empty), and extract the quantiles for each
- * metric. After creating an instance, call metricOption to get the result type.
+ * Helper for getting distributions from nested metric types.
*/
private[v1] abstract class MetricHelper[I, O](
rawMetrics: Seq[InternalTaskMetrics],
quantiles: Array[Double]) {
- def getSubmetrics(raw: InternalTaskMetrics): Option[I]
+ def getSubmetrics(raw: InternalTaskMetrics): I
def build: O
- val data: Seq[I] = rawMetrics.flatMap(getSubmetrics)
+ val data: Seq[I] = rawMetrics.map(getSubmetrics)
/** applies the given function to all input metrics, and returns the quantiles */
def submetricQuantiles(f: I => Double): IndexedSeq[Double] = {
Distribution(data.map { d => f(d) }).get.getQuantiles(quantiles)
}
-
- def metricOption: Option[O] = {
- if (data.isEmpty) {
- None
- } else {
- Some(build)
- }
- }
}
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index ebbbf4814880f3543cf609b9fee6d4d7ac3720f2..ff28796a60f67d4437b59eb20fedc387d8a6a829 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -172,10 +172,10 @@ class TaskMetrics private[spark](
val resultSerializationTime: Long,
val memoryBytesSpilled: Long,
val diskBytesSpilled: Long,
- val inputMetrics: Option[InputMetrics],
- val outputMetrics: Option[OutputMetrics],
- val shuffleReadMetrics: Option[ShuffleReadMetrics],
- val shuffleWriteMetrics: Option[ShuffleWriteMetrics])
+ val inputMetrics: InputMetrics,
+ val outputMetrics: OutputMetrics,
+ val shuffleReadMetrics: ShuffleReadMetrics,
+ val shuffleWriteMetrics: ShuffleWriteMetrics)
class InputMetrics private[spark](
val bytesRead: Long,
@@ -190,7 +190,7 @@ class ShuffleReadMetrics private[spark](
val localBlocksFetched: Int,
val fetchWaitTime: Long,
val remoteBytesRead: Long,
- val totalBlocksFetched: Int,
+ val localBytesRead: Long,
val recordsRead: Long)
class ShuffleWriteMetrics private[spark](
@@ -209,10 +209,10 @@ class TaskMetricDistributions private[spark](
val memoryBytesSpilled: IndexedSeq[Double],
val diskBytesSpilled: IndexedSeq[Double],
- val inputMetrics: Option[InputMetricDistributions],
- val outputMetrics: Option[OutputMetricDistributions],
- val shuffleReadMetrics: Option[ShuffleReadMetricDistributions],
- val shuffleWriteMetrics: Option[ShuffleWriteMetricDistributions])
+ val inputMetrics: InputMetricDistributions,
+ val outputMetrics: OutputMetricDistributions,
+ val shuffleReadMetrics: ShuffleReadMetricDistributions,
+ val shuffleWriteMetrics: ShuffleWriteMetricDistributions)
class InputMetricDistributions private[spark](
val bytesRead: IndexedSeq[Double],
diff --git a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
index 17b33c7c628a30d9aa3473cb7af53cd628c04c71..38ca3224ffe19ff642df396363877bba3126c0b7 100644
--- a/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
+++ b/core/src/main/scala/org/apache/spark/util/JsonProtocol.scala
@@ -326,39 +326,27 @@ private[spark] object JsonProtocol {
}
def taskMetricsToJson(taskMetrics: TaskMetrics): JValue = {
- val shuffleReadMetrics: JValue = if (taskMetrics.shuffleReadMetrics.isUpdated) {
+ val shuffleReadMetrics: JValue =
("Remote Blocks Fetched" -> taskMetrics.shuffleReadMetrics.remoteBlocksFetched) ~
("Local Blocks Fetched" -> taskMetrics.shuffleReadMetrics.localBlocksFetched) ~
("Fetch Wait Time" -> taskMetrics.shuffleReadMetrics.fetchWaitTime) ~
("Remote Bytes Read" -> taskMetrics.shuffleReadMetrics.remoteBytesRead) ~
("Local Bytes Read" -> taskMetrics.shuffleReadMetrics.localBytesRead) ~
("Total Records Read" -> taskMetrics.shuffleReadMetrics.recordsRead)
- } else {
- JNothing
- }
- val shuffleWriteMetrics: JValue = if (taskMetrics.shuffleWriteMetrics.isUpdated) {
+ val shuffleWriteMetrics: JValue =
("Shuffle Bytes Written" -> taskMetrics.shuffleWriteMetrics.bytesWritten) ~
("Shuffle Write Time" -> taskMetrics.shuffleWriteMetrics.writeTime) ~
("Shuffle Records Written" -> taskMetrics.shuffleWriteMetrics.recordsWritten)
- } else {
- JNothing
- }
- val inputMetrics: JValue = if (taskMetrics.inputMetrics.isUpdated) {
+ val inputMetrics: JValue =
("Bytes Read" -> taskMetrics.inputMetrics.bytesRead) ~
("Records Read" -> taskMetrics.inputMetrics.recordsRead)
- } else {
- JNothing
- }
- val outputMetrics: JValue = if (taskMetrics.outputMetrics.isUpdated) {
+ val outputMetrics: JValue =
("Bytes Written" -> taskMetrics.outputMetrics.bytesWritten) ~
("Records Written" -> taskMetrics.outputMetrics.recordsWritten)
- } else {
- JNothing
- }
val updatedBlocks =
JArray(taskMetrics.updatedBlockStatuses.toList.map { case (id, status) =>
("Block ID" -> id.toString) ~
- ("Status" -> blockStatusToJson(status))
+ ("Status" -> blockStatusToJson(status))
})
("Executor Deserialize Time" -> taskMetrics.executorDeserializeTime) ~
("Executor Run Time" -> taskMetrics.executorRunTime) ~
diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
index 1a13233133b1e996eeb10a55493e665c88b8bdb5..cba44c848e0128c30ec1ddbf9b3711de476fbea5 100644
--- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@@ -2,108 +2,108 @@
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917391398,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:11.398GMT",
"lastUpdated" : "",
"duration" : 10505,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1430917381535",
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917380950,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:00.950GMT",
"lastUpdated" : "",
"duration" : 57,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950,
+ "lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
- "startTimeEpoch" : 1430917380880,
- "endTimeEpoch" : 1430917380890,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.880GMT",
"endTime" : "2015-05-06T13:03:00.890GMT",
"lastUpdated" : "",
"duration" : 10,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1426533911241",
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
- "startTimeEpoch" : 1426633910242,
- "endTimeEpoch" : 1426633945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
- "startTimeEpoch" : 1426533910242,
- "endTimeEpoch" : 1426533945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1425081759269",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1425081758277,
- "endTimeEpoch" : 1425081766912,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-28T00:02:38.277GMT",
"endTime" : "2015-02-28T00:02:46.912GMT",
"lastUpdated" : "",
"duration" : 8635,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1425081758277,
+ "endTimeEpoch" : 1425081766912,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981779720,
- "endTimeEpoch" : 1422981788731,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
"duration" : 9011,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981758277,
- "endTimeEpoch" : 1422981766912,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
"duration" : 8635,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0
} ]
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
index 1a13233133b1e996eeb10a55493e665c88b8bdb5..cba44c848e0128c30ec1ddbf9b3711de476fbea5 100644
--- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@@ -2,108 +2,108 @@
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917391398,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:11.398GMT",
"lastUpdated" : "",
"duration" : 10505,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1430917381535",
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917380950,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:00.950GMT",
"lastUpdated" : "",
"duration" : 57,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950,
+ "lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
- "startTimeEpoch" : 1430917380880,
- "endTimeEpoch" : 1430917380890,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.880GMT",
"endTime" : "2015-05-06T13:03:00.890GMT",
"lastUpdated" : "",
"duration" : 10,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1426533911241",
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
- "startTimeEpoch" : 1426633910242,
- "endTimeEpoch" : 1426633945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
- "startTimeEpoch" : 1426533910242,
- "endTimeEpoch" : 1426533945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1425081759269",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1425081758277,
- "endTimeEpoch" : 1425081766912,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-28T00:02:38.277GMT",
"endTime" : "2015-02-28T00:02:46.912GMT",
"lastUpdated" : "",
"duration" : 8635,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1425081758277,
+ "endTimeEpoch" : 1425081766912,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981779720,
- "endTimeEpoch" : 1422981788731,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
"duration" : 9011,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981758277,
- "endTimeEpoch" : 1422981766912,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
"duration" : 8635,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0
} ]
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
index efc865919b0d7d438e56a3d41dc7a8ecc57c8d2a..e7db6742c25e1e4a5515b1652909a1f507328613 100644
--- a/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/executor_list_json_expectation.json
@@ -18,4 +18,4 @@
"totalShuffleWrite" : 13180,
"maxMemory" : 278302556,
"executorLogs" : { }
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
index 2e92e1fa0ec23d847952ace4484d83e678c97a77..bb6bf434be90b11875299300c797aa8cb851cae4 100644
--- a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_1__expectation.json
@@ -12,4 +12,4 @@
"numCompletedStages" : 1,
"numSkippedStages" : 0,
"numFailedStages" : 0
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
index 2e92e1fa0ec23d847952ace4484d83e678c97a77..bb6bf434be90b11875299300c797aa8cb851cae4 100644
--- a/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_from_multi_attempt_app_json_2__expectation.json
@@ -12,4 +12,4 @@
"numCompletedStages" : 1,
"numSkippedStages" : 0,
"numFailedStages" : 0
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
index cab4750270dfa31e5019a6aff24544187f12770a..1583e5ddef565d5aaa1c1e64dbef03b182854813 100644
--- a/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/job_list_json_expectation.json
@@ -40,4 +40,4 @@
"numCompletedStages" : 1,
"numSkippedStages" : 0,
"numFailedStages" : 0
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
index eacf04b9016ac251be5059f135f82b7c284c5bce..a525d61543a88691a45cacf268872adafe4fdeb0 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
@@ -2,14 +2,14 @@
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981758277,
- "endTimeEpoch" : 1422981766912,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
"duration" : 8635,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0
} ]
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
index adad25bf17fd53ccd7b89076ed93f17d1c493b3b..cc567f66f02e80865cb2fe558befe8bb2ded70e0 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
@@ -2,28 +2,28 @@
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981779720,
- "endTimeEpoch" : 1422981788731,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
"duration" : 9011,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1422981759269",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981758277,
- "endTimeEpoch" : 1422981766912,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:38.277GMT",
"endTime" : "2015-02-03T16:42:46.912GMT",
"lastUpdated" : "",
"duration" : 8635,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981758277,
+ "endTimeEpoch" : 1422981766912,
+ "lastUpdatedEpoch" : 0
} ]
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
index a658909088a4afd222fa9deacd1987f3ddaa728b..c934a871724b59b28eb05239481bf2035d745b98 100644
--- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@@ -2,82 +2,80 @@
"id" : "local-1430917381534",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917391398,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:11.398GMT",
"lastUpdated" : "",
"duration" : 10505,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917391398,
+ "lastUpdatedEpoch" : 0
} ]
-}, {
+}, {
"id" : "local-1430917381535",
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
- "startTimeEpoch" : 1430917380893,
- "endTimeEpoch" : 1430917380950,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.893GMT",
"endTime" : "2015-05-06T13:03:00.950GMT",
"lastUpdated" : "",
"duration" : 57,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380893,
+ "endTimeEpoch" : 1430917380950,
+ "lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
- "startTimeEpoch" : 1430917380880,
- "endTimeEpoch" : 1430917380890,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-05-06T13:03:00.880GMT",
"endTime" : "2015-05-06T13:03:00.890GMT",
"lastUpdated" : "",
"duration" : 10,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1430917380880,
+ "endTimeEpoch" : 1430917380890,
+ "lastUpdatedEpoch" : 0
} ]
}, {
"id" : "local-1426533911241",
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
- "startTimeEpoch" : 1426633910242,
- "endTimeEpoch" : 1426633945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
- "startTimeEpoch" : 1426533910242,
- "endTimeEpoch" : 1426533945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0
} ]
}, {
- "id": "local-1425081759269",
- "name": "Spark shell",
- "attempts": [
- {
- "startTimeEpoch" : 1425081758277,
- "endTimeEpoch" : 1425081766912,
- "lastUpdatedEpoch" : 0,
- "startTime": "2015-02-28T00:02:38.277GMT",
- "endTime": "2015-02-28T00:02:46.912GMT",
- "lastUpdated" : "",
- "duration" : 8635,
- "sparkUser": "irashid",
- "completed": true
- }
- ]
+ "id" : "local-1425081759269",
+ "name" : "Spark shell",
+ "attempts" : [ {
+ "startTime" : "2015-02-28T00:02:38.277GMT",
+ "endTime" : "2015-02-28T00:02:46.912GMT",
+ "lastUpdated" : "",
+ "duration" : 8635,
+ "sparkUser" : "irashid",
+ "completed" : true,
+ "startTimeEpoch" : 1425081758277,
+ "endTimeEpoch" : 1425081766912,
+ "lastUpdatedEpoch" : 0
+ } ]
} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
index 0217facad9ded8edf7ae2ae349aeb657e153c5f8..f486d46313d8b9d12c4fac2e0efdf2ded453ddb9 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
@@ -2,14 +2,14 @@
"id" : "local-1422981780767",
"name" : "Spark shell",
"attempts" : [ {
- "startTimeEpoch" : 1422981779720,
- "endTimeEpoch" : 1422981788731,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-02-03T16:42:59.720GMT",
"endTime" : "2015-02-03T16:43:08.731GMT",
"lastUpdated" : "",
"duration" : 9011,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1422981779720,
+ "endTimeEpoch" : 1422981788731,
+ "lastUpdatedEpoch" : 0
} ]
}
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
index b20a26648e43054e78fe42ba9f559b729e5185d7..e63039f6a17fc4cfb18ef10082e440e91dd4d672 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
@@ -3,25 +3,25 @@
"name" : "Spark shell",
"attempts" : [ {
"attemptId" : "2",
- "startTimeEpoch" : 1426633910242,
- "endTimeEpoch" : 1426633945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-17T23:11:50.242GMT",
"endTime" : "2015-03-17T23:12:25.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426633910242,
+ "endTimeEpoch" : 1426633945177,
+ "lastUpdatedEpoch" : 0
}, {
"attemptId" : "1",
- "startTimeEpoch" : 1426533910242,
- "endTimeEpoch" : 1426533945177,
- "lastUpdatedEpoch" : 0,
"startTime" : "2015-03-16T19:25:10.242GMT",
"endTime" : "2015-03-16T19:25:45.177GMT",
"lastUpdated" : "",
"duration" : 34935,
"sparkUser" : "irashid",
- "completed" : true
+ "completed" : true,
+ "startTimeEpoch" : 1426533910242,
+ "endTimeEpoch" : 1426533945177,
+ "lastUpdatedEpoch" : 0
} ]
}
diff --git a/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
index 4a29072bdb6e484338e0b291a8c08dca1ec9f9f1..f1f0ec885587b2d4f4cf19a8e8cc2087b7ee3704 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_job_json_expectation.json
@@ -12,4 +12,4 @@
"numCompletedStages" : 1,
"numSkippedStages" : 0,
"numFailedStages" : 0
-}
\ No newline at end of file
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/one_rdd_storage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_rdd_storage_json_expectation.json
deleted file mode 100644
index 38b5328ffbb03638e47ca41ddde8f65802b76cd5..0000000000000000000000000000000000000000
--- a/core/src/test/resources/HistoryServerExpectations/one_rdd_storage_json_expectation.json
+++ /dev/null
@@ -1,64 +0,0 @@
-{
- "id" : 0,
- "name" : "0",
- "numPartitions" : 8,
- "numCachedPartitions" : 8,
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 28000128,
- "diskUsed" : 0,
- "dataDistribution" : [ {
- "address" : "localhost:57971",
- "memoryUsed" : 28000128,
- "memoryRemaining" : 250302428,
- "diskUsed" : 0
- } ],
- "partitions" : [ {
- "blockName" : "rdd_0_0",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- }, {
- "blockName" : "rdd_0_1",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- }, {
- "blockName" : "rdd_0_2",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- }, {
- "blockName" : "rdd_0_3",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- }, {
- "blockName" : "rdd_0_4",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- }, {
- "blockName" : "rdd_0_5",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- }, {
- "blockName" : "rdd_0_6",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- }, {
- "blockName" : "rdd_0_7",
- "storageLevel" : "Memory Deserialized 1x Replicated",
- "memoryUsed" : 3500016,
- "diskUsed" : 0,
- "executors" : [ "localhost:57971" ]
- } ]
-}
\ No newline at end of file
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
index b07011d4f113f53a2bd9b61e9d3b22bfbe61eb44..477a2fec8b69b8b8e1cb10f8cfc5eef63add6f08 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_attempt_json_expectation.json
@@ -46,6 +46,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 94000,
@@ -75,6 +87,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1647,
"writeTime" : 83000,
@@ -104,6 +128,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 88000,
@@ -133,6 +169,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 73000,
@@ -162,6 +210,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 76000,
@@ -191,6 +251,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 98000,
@@ -220,6 +292,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1645,
"writeTime" : 101000,
@@ -249,6 +333,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 79000,
diff --git a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
index 2f71520549e1f86557b09b0f015cebdbab32073f..388e51f77a24d0f8bc2cd025928f7bedaa967454 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_stage_json_expectation.json
@@ -46,6 +46,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 94000,
@@ -75,6 +87,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1647,
"writeTime" : 83000,
@@ -104,6 +128,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 88000,
@@ -133,6 +169,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 73000,
@@ -162,6 +210,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 76000,
@@ -191,6 +251,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 98000,
@@ -220,6 +292,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1645,
"writeTime" : 101000,
@@ -249,6 +333,18 @@
"bytesRead" : 3500016,
"recordsRead" : 0
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1648,
"writeTime" : 79000,
diff --git a/core/src/test/resources/HistoryServerExpectations/rdd_list_storage_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/rdd_list_storage_json_expectation.json
index 8878e547a7984ed8b4aac1fe1a023e070e6489a0..1e3ec7217afba05e6dcd456b0e89ec7f17bf7b55 100644
--- a/core/src/test/resources/HistoryServerExpectations/rdd_list_storage_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/rdd_list_storage_json_expectation.json
@@ -1 +1 @@
-[ ]
\ No newline at end of file
+[ ]
diff --git a/core/src/test/resources/HistoryServerExpectations/running_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/running_app_list_json_expectation.json
index 8878e547a7984ed8b4aac1fe1a023e070e6489a0..1e3ec7217afba05e6dcd456b0e89ec7f17bf7b55 100644
--- a/core/src/test/resources/HistoryServerExpectations/running_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/running_app_list_json_expectation.json
@@ -1 +1 @@
-[ ]
\ No newline at end of file
+[ ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
index f2cb29b31c85f2ad1ab35019223f9cff25b1525e..8e09aabbad7c945f2177d3af0972e9c9582766ce 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_expectation.json
@@ -20,6 +20,18 @@
"bytesRead" : 49294,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3842811,
@@ -48,6 +60,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3934399,
@@ -76,6 +100,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 89885,
@@ -104,6 +140,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 1311694,
@@ -132,6 +180,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 83022,
@@ -160,6 +220,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3675510,
@@ -188,6 +260,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 4016617,
@@ -216,6 +300,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 2579051,
@@ -244,6 +340,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 121551,
@@ -272,6 +380,18 @@
"bytesRead" : 60489,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 101664,
@@ -300,6 +420,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94709,
@@ -328,6 +460,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94507,
@@ -356,6 +500,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 102476,
@@ -384,6 +540,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95004,
@@ -412,6 +580,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95646,
@@ -440,6 +620,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 602780,
@@ -468,6 +660,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 108320,
@@ -496,6 +700,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 99944,
@@ -524,6 +740,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100836,
@@ -552,10 +780,22 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95788,
"recordsWritten" : 10
}
}
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
index c3febc5fc9447bc46952cbed6e00d6229e353175..1dbf72b42a92636389482665d88cc78fa90af757 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_1__expectation.json
@@ -20,7 +20,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 1,
@@ -44,7 +65,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 2,
@@ -68,7 +110,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 3,
@@ -92,7 +155,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 4,
@@ -116,7 +200,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 5,
@@ -140,7 +245,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 6,
@@ -164,7 +290,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 7,
@@ -188,6 +335,27 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
index 56d667d88917cff8f480f972b19cbd2237dad38e..483492282dd6469d727dd733bb0f19593991d72d 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_from_multi_attempt_app_json_2__expectation.json
@@ -20,7 +20,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 1,
@@ -44,7 +65,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 2,
@@ -68,7 +110,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 3,
@@ -92,7 +155,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 4,
@@ -116,7 +200,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 5,
@@ -140,7 +245,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 6,
@@ -164,7 +290,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}, {
"taskId" : 7,
@@ -188,6 +335,27 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
index e5ec3bc4c7126d4daa84d40065bd4187ec834836..624f2bb16df482d59bb7d02e10016f08bd55dd1c 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__offset___length_expectation.json
@@ -20,6 +20,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94709,
@@ -48,6 +60,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94507,
@@ -76,6 +100,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 102476,
@@ -104,6 +140,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95004,
@@ -132,6 +180,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95646,
@@ -160,6 +220,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 602780,
@@ -188,6 +260,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 108320,
@@ -216,6 +300,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 99944,
@@ -244,6 +340,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100836,
@@ -272,6 +380,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95788,
@@ -300,6 +420,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 97716,
@@ -328,6 +460,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100270,
@@ -356,6 +500,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 143427,
@@ -384,6 +540,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 91844,
@@ -412,6 +580,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 157194,
@@ -440,6 +620,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94134,
@@ -468,6 +660,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 108213,
@@ -496,6 +700,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 102019,
@@ -524,6 +740,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 104299,
@@ -552,6 +780,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 114938,
@@ -580,6 +820,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 119770,
@@ -608,6 +860,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 92619,
@@ -636,6 +900,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 89603,
@@ -664,6 +940,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 118329,
@@ -692,6 +980,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 127746,
@@ -720,6 +1020,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 160963,
@@ -748,6 +1060,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 123855,
@@ -776,6 +1100,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 111869,
@@ -804,6 +1140,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 131158,
@@ -832,6 +1180,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 98748,
@@ -860,6 +1220,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94792,
@@ -888,6 +1260,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 90765,
@@ -916,6 +1300,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 103713,
@@ -944,6 +1340,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 171516,
@@ -972,6 +1380,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 98293,
@@ -1000,6 +1420,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 92985,
@@ -1028,6 +1460,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 113322,
@@ -1056,6 +1500,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 103015,
@@ -1084,6 +1540,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 139844,
@@ -1112,6 +1580,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94984,
@@ -1140,6 +1620,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 90836,
@@ -1168,6 +1660,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 96013,
@@ -1196,6 +1700,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 89664,
@@ -1224,6 +1740,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 92835,
@@ -1252,6 +1780,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 90506,
@@ -1280,6 +1820,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 108309,
@@ -1308,6 +1860,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 90329,
@@ -1336,6 +1900,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 96849,
@@ -1364,6 +1940,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 97521,
@@ -1392,10 +1980,22 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100753,
"recordsWritten" : 10
}
}
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
index 5657123a2db15df37ecd6820f31753d6cf9583ba..11eec0b49c40b16f5024d2540d0886cf8cc5a885 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_expectation.json
@@ -20,6 +20,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 4016617,
@@ -48,6 +60,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3675510,
@@ -76,6 +100,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3934399,
@@ -104,6 +140,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 83022,
@@ -132,6 +180,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 2579051,
@@ -160,6 +220,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 1311694,
@@ -188,6 +260,18 @@
"bytesRead" : 49294,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3842811,
@@ -216,6 +300,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 89885,
@@ -244,6 +340,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 143427,
@@ -272,6 +380,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100836,
@@ -300,6 +420,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 99944,
@@ -328,6 +460,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100270,
@@ -356,6 +500,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 108320,
@@ -384,6 +540,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95788,
@@ -412,6 +580,18 @@
"bytesRead" : 60489,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 101664,
@@ -440,6 +620,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 97716,
@@ -468,6 +660,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95646,
@@ -496,6 +700,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 121551,
@@ -524,6 +740,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 102476,
@@ -552,10 +780,22 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95004,
"recordsWritten" : 10
}
}
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
index 5657123a2db15df37ecd6820f31753d6cf9583ba..11eec0b49c40b16f5024d2540d0886cf8cc5a885 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names___runtime_expectation.json
@@ -20,6 +20,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 4016617,
@@ -48,6 +60,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3675510,
@@ -76,6 +100,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3934399,
@@ -104,6 +140,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 83022,
@@ -132,6 +180,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 2579051,
@@ -160,6 +220,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 1311694,
@@ -188,6 +260,18 @@
"bytesRead" : 49294,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 3842811,
@@ -216,6 +300,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 89885,
@@ -244,6 +340,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 143427,
@@ -272,6 +380,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100836,
@@ -300,6 +420,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 99944,
@@ -328,6 +460,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100270,
@@ -356,6 +500,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 108320,
@@ -384,6 +540,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95788,
@@ -412,6 +580,18 @@
"bytesRead" : 60489,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 101664,
@@ -440,6 +620,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 97716,
@@ -468,6 +660,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95646,
@@ -496,6 +700,18 @@
"bytesRead" : 60488,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 121551,
@@ -524,6 +740,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 102476,
@@ -552,10 +780,22 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95004,
"recordsWritten" : 10
}
}
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
index 72fe017e9f85d71dff4840395424c3a58c97a759..9528d872ef731083eb1cdc0ccf2675315739da14 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_list_w__sortBy_short_names__runtime_expectation.json
@@ -20,6 +20,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 94792,
@@ -48,6 +60,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 95848,
@@ -76,6 +100,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 90765,
@@ -104,6 +140,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 101750,
@@ -132,6 +180,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 97521,
@@ -160,6 +220,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 171516,
@@ -188,6 +260,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 96849,
@@ -216,6 +300,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 100753,
@@ -244,6 +340,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 89603,
@@ -272,6 +380,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 102159,
@@ -300,6 +420,18 @@
"bytesRead" : 70565,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 133964,
@@ -328,6 +460,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 102779,
@@ -356,6 +500,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 98472,
@@ -384,6 +540,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 98748,
@@ -412,6 +580,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 103713,
@@ -440,6 +620,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 96013,
@@ -468,6 +660,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 90836,
@@ -496,6 +700,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 92835,
@@ -524,6 +740,18 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 98293,
@@ -552,10 +780,22 @@
"bytesRead" : 70564,
"recordsRead" : 10000
},
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
"shuffleWriteMetrics" : {
"bytesWritten" : 1710,
"writeTime" : 98069,
"recordsWritten" : 10
}
}
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
index bc3c302813de249a7560c529c65cdaa55c24a387..76d1553bc8f77441845fdd83876819a7d26c15b3 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w__custom_quantiles_expectation.json
@@ -11,9 +11,22 @@
"bytesRead" : [ 60488.0, 70564.0, 70565.0 ],
"recordsRead" : [ 10000.0, 10000.0, 10000.0 ]
},
+ "outputMetrics" : {
+ "bytesWritten" : [ 0.0, 0.0, 0.0 ],
+ "recordsWritten" : [ 0.0, 0.0, 0.0 ]
+ },
+ "shuffleReadMetrics" : {
+ "readBytes" : [ 0.0, 0.0, 0.0 ],
+ "readRecords" : [ 0.0, 0.0, 0.0 ],
+ "remoteBlocksFetched" : [ 0.0, 0.0, 0.0 ],
+ "localBlocksFetched" : [ 0.0, 0.0, 0.0 ],
+ "fetchWaitTime" : [ 0.0, 0.0, 0.0 ],
+ "remoteBytesRead" : [ 0.0, 0.0, 0.0 ],
+ "totalBlocksFetched" : [ 0.0, 0.0, 0.0 ]
+ },
"shuffleWriteMetrics" : {
"writeBytes" : [ 1710.0, 1710.0, 1710.0 ],
"writeRecords" : [ 10.0, 10.0, 10.0 ],
"writeTime" : [ 89437.0, 102159.0, 4016617.0 ]
}
-}
\ No newline at end of file
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
index e084c839f1d5a9388b6de426c465d58c27c5e7a6..7baffc5df0b0f566bef7d6224be0a2c65de704f5 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_read_expectation.json
@@ -7,6 +7,14 @@
"resultSerializationTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"memoryBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"diskBytesSpilled" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "inputMetrics" : {
+ "bytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "recordsRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+ },
+ "outputMetrics" : {
+ "bytesWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "recordsWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+ },
"shuffleReadMetrics" : {
"readBytes" : [ 17100.0, 17100.0, 17100.0, 17100.0, 17100.0 ],
"readRecords" : [ 100.0, 100.0, 100.0, 100.0, 100.0 ],
@@ -15,5 +23,10 @@
"fetchWaitTime" : [ 0.0, 0.0, 0.0, 1.0, 1.0 ],
"remoteBytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
"totalBlocksFetched" : [ 100.0, 100.0, 100.0, 100.0, 100.0 ]
+ },
+ "shuffleWriteMetrics" : {
+ "writeBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "writeRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "writeTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
}
-}
\ No newline at end of file
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
index 6ac7811ce691b06be3eea22716b1f413f067ad44..f8c4b7c12873359807887b1772997b7153a3392d 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_task_summary_w_shuffle_write_expectation.json
@@ -11,9 +11,22 @@
"bytesRead" : [ 60488.0, 70564.0, 70564.0, 70564.0, 70564.0 ],
"recordsRead" : [ 10000.0, 10000.0, 10000.0, 10000.0, 10000.0 ]
},
+ "outputMetrics" : {
+ "bytesWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "recordsWritten" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+ },
+ "shuffleReadMetrics" : {
+ "readBytes" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "readRecords" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "remoteBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "localBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "fetchWaitTime" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "remoteBytesRead" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ],
+ "totalBlocksFetched" : [ 0.0, 0.0, 0.0, 0.0, 0.0 ]
+ },
"shuffleWriteMetrics" : {
"writeBytes" : [ 1710.0, 1710.0, 1710.0, 1710.0, 1710.0 ],
"writeRecords" : [ 10.0, 10.0, 10.0, 10.0, 10.0 ],
"writeTime" : [ 90329.0, 95848.0, 102159.0, 121551.0, 2579051.0 ]
}
-}
\ No newline at end of file
+}
diff --git a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
index 12665a152c9ec9b047dbc841ad7d34b7d557a8cc..ce008bf40967d17526df0c6ed50905511616443b 100644
--- a/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/stage_with_accumulable_json_expectation.json
@@ -50,7 +50,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
},
"5" : {
@@ -75,7 +96,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
},
"4" : {
@@ -100,7 +142,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 1,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
},
"7" : {
@@ -125,7 +188,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
},
"1" : {
@@ -150,7 +234,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
},
"3" : {
@@ -175,7 +280,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
},
"6" : {
@@ -200,7 +326,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
},
"0" : {
@@ -225,7 +372,28 @@
"jvmGcTime" : 0,
"resultSerializationTime" : 2,
"memoryBytesSpilled" : 0,
- "diskBytesSpilled" : 0
+ "diskBytesSpilled" : 0,
+ "inputMetrics" : {
+ "bytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "outputMetrics" : {
+ "bytesWritten" : 0,
+ "recordsWritten" : 0
+ },
+ "shuffleReadMetrics" : {
+ "remoteBlocksFetched" : 0,
+ "localBlocksFetched" : 0,
+ "fetchWaitTime" : 0,
+ "remoteBytesRead" : 0,
+ "localBytesRead" : 0,
+ "recordsRead" : 0
+ },
+ "shuffleWriteMetrics" : {
+ "bytesWritten" : 0,
+ "writeTime" : 0,
+ "recordsWritten" : 0
+ }
}
}
},
diff --git a/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
index cab4750270dfa31e5019a6aff24544187f12770a..1583e5ddef565d5aaa1c1e64dbef03b182854813 100644
--- a/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/succeeded_failed_job_list_json_expectation.json
@@ -40,4 +40,4 @@
"numCompletedStages" : 1,
"numSkippedStages" : 0,
"numFailedStages" : 0
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
index 6fd25befbf7e8b56d0597afdecc5adf4889f6aa6..c232c983237559f7f2d4fd6267e4da9e2465fe12 100644
--- a/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/succeeded_job_list_json_expectation.json
@@ -26,4 +26,4 @@
"numCompletedStages" : 1,
"numSkippedStages" : 0,
"numFailedStages" : 0
-} ]
\ No newline at end of file
+} ]
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
index 2a013aca7b89564282fd9cc8546649b616e8a18d..631a7cd9d5d7a4be4e25016044f3d69b459e3c8a 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/HistoryServerSuite.scala
@@ -153,37 +153,39 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
code should be (HttpServletResponse.SC_OK)
jsonOpt should be ('defined)
errOpt should be (None)
- val jsonOrg = jsonOpt.get
-
- // SPARK-10873 added the lastUpdated field for each application's attempt,
- // the REST API returns the last modified time of EVENT LOG file for this field.
- // It is not applicable to hard-code this dynamic field in a static expected file,
- // so here we skip checking the lastUpdated field's value (setting it as "").
- val json = if (jsonOrg.indexOf("lastUpdated") >= 0) {
- val subStrings = jsonOrg.split(",")
- for (i <- subStrings.indices) {
- if (subStrings(i).indexOf("lastUpdatedEpoch") >= 0) {
- subStrings(i) = subStrings(i).replaceAll("(\\d+)", "0")
- } else if (subStrings(i).indexOf("lastUpdated") >= 0) {
- subStrings(i) = "\"lastUpdated\":\"\""
- }
- }
- subStrings.mkString(",")
- } else {
- jsonOrg
- }
val exp = IOUtils.toString(new FileInputStream(
new File(expRoot, HistoryServerSuite.sanitizePath(name) + "_expectation.json")))
// compare the ASTs so formatting differences don't cause failures
import org.json4s._
import org.json4s.jackson.JsonMethods._
- val jsonAst = parse(json)
+ val jsonAst = parse(clearLastUpdated(jsonOpt.get))
val expAst = parse(exp)
assertValidDataInJson(jsonAst, expAst)
}
}
+ // SPARK-10873 added the lastUpdated field for each application's attempt,
+ // the REST API returns the last modified time of EVENT LOG file for this field.
+ // It is not applicable to hard-code this dynamic field in a static expected file,
+ // so here we skip checking the lastUpdated field's value (setting it as "").
+ private def clearLastUpdated(json: String): String = {
+ if (json.indexOf("lastUpdated") >= 0) {
+ val subStrings = json.split(",")
+ for (i <- subStrings.indices) {
+ if (subStrings(i).indexOf("lastUpdatedEpoch") >= 0) {
+ subStrings(i) = subStrings(i).replaceAll("(\\d+)", "0")
+ } else if (subStrings(i).indexOf("lastUpdated") >= 0) {
+ val regex = "\"lastUpdated\"\\s*:\\s*\".*\"".r
+ subStrings(i) = regex.replaceAllIn(subStrings(i), "\"lastUpdated\" : \"\"")
+ }
+ }
+ subStrings.mkString(",")
+ } else {
+ json
+ }
+ }
+
test("download all logs for app with multiple attempts") {
doDownloadTest("local-1430917381535", None)
}
@@ -486,7 +488,8 @@ class HistoryServerSuite extends SparkFunSuite with BeforeAndAfter with Matchers
val json = getUrl(path)
val file = new File(expRoot, HistoryServerSuite.sanitizePath(name) + "_expectation.json")
val out = new FileWriter(file)
- out.write(json)
+ out.write(clearLastUpdated(json))
+ out.write('\n')
out.close()
}
diff --git a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
index 612c7c1954aabfaf74bf8743200eb14bf3327ce7..d3b6cdfe86eeccebfcf4939e41022b3b6ffe20da 100644
--- a/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
+++ b/core/src/test/scala/org/apache/spark/util/JsonProtocolSuite.scala
@@ -1111,6 +1111,14 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Shuffle Write Time": 1500,
| "Shuffle Records Written": 12
| },
+ | "Input Metrics" : {
+ | "Bytes Read" : 0,
+ | "Records Read" : 0
+ | },
+ | "Output Metrics" : {
+ | "Bytes Written" : 0,
+ | "Records Written" : 0
+ | },
| "Updated Blocks": [
| {
| "Block ID": "rdd_0_0",
@@ -1187,6 +1195,14 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Result Serialization Time": 700,
| "Memory Bytes Spilled": 800,
| "Disk Bytes Spilled": 0,
+ | "Shuffle Read Metrics" : {
+ | "Remote Blocks Fetched" : 0,
+ | "Local Blocks Fetched" : 0,
+ | "Fetch Wait Time" : 0,
+ | "Remote Bytes Read" : 0,
+ | "Local Bytes Read" : 0,
+ | "Total Records Read" : 0
+ | },
| "Shuffle Write Metrics": {
| "Shuffle Bytes Written": 1200,
| "Shuffle Write Time": 1500,
@@ -1196,6 +1212,10 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Bytes Read": 2100,
| "Records Read": 21
| },
+ | "Output Metrics" : {
+ | "Bytes Written" : 0,
+ | "Records Written" : 0
+ | },
| "Updated Blocks": [
| {
| "Block ID": "rdd_0_0",
@@ -1272,6 +1292,19 @@ private[spark] object JsonProtocolSuite extends Assertions {
| "Result Serialization Time": 700,
| "Memory Bytes Spilled": 800,
| "Disk Bytes Spilled": 0,
+ | "Shuffle Read Metrics" : {
+ | "Remote Blocks Fetched" : 0,
+ | "Local Blocks Fetched" : 0,
+ | "Fetch Wait Time" : 0,
+ | "Remote Bytes Read" : 0,
+ | "Local Bytes Read" : 0,
+ | "Total Records Read" : 0
+ | },
+ | "Shuffle Write Metrics" : {
+ | "Shuffle Bytes Written" : 0,
+ | "Shuffle Write Time" : 0,
+ | "Shuffle Records Written" : 0
+ | },
| "Input Metrics": {
| "Bytes Read": 2100,
| "Records Read": 21
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 7730823f9411bb095926a7bd52e0a0ba57b505b5..ff35dc010dc2f7ff73a3f726092724b006b299fd 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -634,6 +634,20 @@ object MimaExcludes {
// [SPARK-14628] Simplify task metrics by always tracking read/write metrics
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.InputMetrics.readMethod"),
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.executor.OutputMetrics.writeMethod")
+ ) ++ Seq(
+ // SPARK-14628: Always track input/output/shuffle metrics
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.totalBlocksFetched"),
+ ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.status.api.v1.ShuffleReadMetrics.this"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.inputMetrics"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.outputMetrics"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.shuffleWriteMetrics"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.shuffleReadMetrics"),
+ ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.status.api.v1.TaskMetrics.this"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.inputMetrics"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.outputMetrics"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.shuffleWriteMetrics"),
+ ProblemFilters.exclude[IncompatibleResultTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.shuffleReadMetrics"),
+ ProblemFilters.exclude[IncompatibleMethTypeProblem]("org.apache.spark.status.api.v1.TaskMetricDistributions.this")
)
case v if v.startsWith("1.6") =>
Seq(