diff --git a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
index 6d8758a3d3b1df045bf80d33b6094674d2e6a8d3..5cb48ca3e60b0edad5e782e55221ae5b45155a36 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/ApplicationHistoryProvider.scala
@@ -30,7 +30,8 @@ private[spark] case class ApplicationAttemptInfo(
endTime: Long,
lastUpdated: Long,
sparkUser: String,
- completed: Boolean = false)
+ completed: Boolean = false,
+ appSparkVersion: String)
private[spark] case class ApplicationHistoryInfo(
id: String,
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index f4235df2451282b8324ae5e439384ce93dbcb11a..d05ca142b618bc6757074e62619ecb0ab22c65d8 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -248,7 +248,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
val conf = this.conf.clone()
val appSecManager = new SecurityManager(conf)
SparkUI.createHistoryUI(conf, replayBus, appSecManager, appInfo.name,
- HistoryServer.getAttemptURI(appId, attempt.attemptId), attempt.startTime)
+ HistoryServer.getAttemptURI(appId, attempt.attemptId),
+ attempt.startTime)
// Do not call ui.bind() to avoid creating a new server for each application
}
@@ -257,6 +258,7 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
val appListener = replay(fileStatus, isApplicationCompleted(fileStatus), replayBus)
if (appListener.appId.isDefined) {
+ ui.appSparkVersion = appListener.appSparkVersion.getOrElse("")
ui.getSecurityManager.setAcls(HISTORY_UI_ACLS_ENABLE)
// make sure to set admin acls before view acls so they are properly picked up
val adminAcls = HISTORY_UI_ADMIN_ACLS + "," + appListener.adminAcls.getOrElse("")
@@ -443,7 +445,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
val newAttempts = try {
val eventsFilter: ReplayEventsFilter = { eventString =>
eventString.startsWith(APPL_START_EVENT_PREFIX) ||
- eventString.startsWith(APPL_END_EVENT_PREFIX)
+ eventString.startsWith(APPL_END_EVENT_PREFIX) ||
+ eventString.startsWith(LOG_START_EVENT_PREFIX)
}
val logPath = fileStatus.getPath()
@@ -469,7 +472,8 @@ private[history] class FsHistoryProvider(conf: SparkConf, clock: Clock)
lastUpdated,
appListener.sparkUser.getOrElse(NOT_STARTED),
appCompleted,
- fileStatus.getLen()
+ fileStatus.getLen(),
+ appListener.appSparkVersion.getOrElse("")
)
fileToAppInfo(logPath) = attemptInfo
logDebug(s"Application log ${attemptInfo.logPath} loaded successfully: $attemptInfo")
@@ -735,6 +739,8 @@ private[history] object FsHistoryProvider {
private val APPL_START_EVENT_PREFIX = "{\"Event\":\"SparkListenerApplicationStart\""
private val APPL_END_EVENT_PREFIX = "{\"Event\":\"SparkListenerApplicationEnd\""
+
+ private val LOG_START_EVENT_PREFIX = "{\"Event\":\"SparkListenerLogStart\""
}
/**
@@ -762,9 +768,10 @@ private class FsApplicationAttemptInfo(
lastUpdated: Long,
sparkUser: String,
completed: Boolean,
- val fileSize: Long)
+ val fileSize: Long,
+ appSparkVersion: String)
extends ApplicationAttemptInfo(
- attemptId, startTime, endTime, lastUpdated, sparkUser, completed) {
+ attemptId, startTime, endTime, lastUpdated, sparkUser, completed, appSparkVersion) {
/** extend the superclass string value with the extra attributes of this class */
override def toString: String = {
diff --git a/core/src/main/scala/org/apache/spark/scheduler/ApplicationEventListener.scala b/core/src/main/scala/org/apache/spark/scheduler/ApplicationEventListener.scala
index 28c45d800ed0671dc33323d92c8d3a13faf91028..6da8865cd10d3ea2b4ba793c1e9b62ef956c3efb 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/ApplicationEventListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/ApplicationEventListener.scala
@@ -34,6 +34,7 @@ private[spark] class ApplicationEventListener extends SparkListener {
var adminAcls: Option[String] = None
var viewAclsGroups: Option[String] = None
var adminAclsGroups: Option[String] = None
+ var appSparkVersion: Option[String] = None
override def onApplicationStart(applicationStart: SparkListenerApplicationStart) {
appName = Some(applicationStart.appName)
@@ -57,4 +58,10 @@ private[spark] class ApplicationEventListener extends SparkListener {
adminAclsGroups = allProperties.get("spark.admin.acls.groups")
}
}
+
+ override def onOtherEvent(event: SparkListenerEvent): Unit = event match {
+ case SparkListenerLogStart(sparkVersion) =>
+ appSparkVersion = Some(sparkVersion)
+ case _ =>
+ }
}
diff --git a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
index a7dbf87915b27a5c8eb21f7e7f201cd7aa494ced..f481436332249638c7ea7a9a5cb807df3ccbdeed 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/EventLoggingListener.scala
@@ -119,7 +119,7 @@ private[spark] class EventLoggingListener(
val cstream = compressionCodec.map(_.compressedOutputStream(dstream)).getOrElse(dstream)
val bstream = new BufferedOutputStream(cstream, outputBufferSize)
- EventLoggingListener.initEventLog(bstream)
+ EventLoggingListener.initEventLog(bstream, testing, loggedEvents)
fileSystem.setPermission(path, LOG_FILE_PERMISSIONS)
writer = Some(new PrintWriter(bstream))
logInfo("Logging events to %s".format(logPath))
@@ -283,10 +283,17 @@ private[spark] object EventLoggingListener extends Logging {
*
* @param logStream Raw output stream to the event log file.
*/
- def initEventLog(logStream: OutputStream): Unit = {
+ def initEventLog(
+ logStream: OutputStream,
+ testing: Boolean,
+ loggedEvents: ArrayBuffer[JValue]): Unit = {
val metadata = SparkListenerLogStart(SPARK_VERSION)
- val metadataJson = compact(JsonProtocol.logStartToJson(metadata)) + "\n"
+ val eventJson = JsonProtocol.logStartToJson(metadata)
+ val metadataJson = compact(eventJson) + "\n"
logStream.write(metadataJson.getBytes(StandardCharsets.UTF_8))
+ if (testing && loggedEvents != null) {
+ loggedEvents += eventJson
+ }
}
/**
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
index bc2e53071668671bc3fa9d8ca7304979cbfd1fc4..59f89a82a1da85d563d81625b045bd85f8b9af73 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListener.scala
@@ -160,9 +160,9 @@ case class SparkListenerApplicationEnd(time: Long) extends SparkListenerEvent
/**
* An internal class that describes the metadata of an event log.
- * This event is not meant to be posted to listeners downstream.
*/
-private[spark] case class SparkListenerLogStart(sparkVersion: String) extends SparkListenerEvent
+@DeveloperApi
+case class SparkListenerLogStart(sparkVersion: String) extends SparkListenerEvent
/**
* Interface for creating history listeners defined in other modules like SQL, which are used to
diff --git a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
index 3ff363321e8c9cf42ed8e60c66641560ea80e354..3b0d3b1b150feaf779e3eabe1d48b740d3602771 100644
--- a/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
+++ b/core/src/main/scala/org/apache/spark/scheduler/SparkListenerBus.scala
@@ -71,7 +71,6 @@ private[spark] trait SparkListenerBus
listener.onNodeUnblacklisted(nodeUnblacklisted)
case blockUpdated: SparkListenerBlockUpdated =>
listener.onBlockUpdated(blockUpdated)
- case logStart: SparkListenerLogStart => // ignore event log metadata
case _ => listener.onOtherEvent(event)
}
}
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
index a0239266d8756660c97de25833d7275f7a58b2bb..f039744e7f67fb1ffa34ce238127f28be8542a74 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/ApplicationListResource.scala
@@ -90,7 +90,8 @@ private[spark] object ApplicationsListResource {
},
lastUpdated = new Date(internalAttemptInfo.lastUpdated),
sparkUser = internalAttemptInfo.sparkUser,
- completed = internalAttemptInfo.completed
+ completed = internalAttemptInfo.completed,
+ appSparkVersion = internalAttemptInfo.appSparkVersion
)
}
)
diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
index 56d8e51732ffd04fe0693e48b278ca9baaf23b66..f6203271f3cd275c864ed61ab5d87516b93b2d0f 100644
--- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
+++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala
@@ -38,7 +38,8 @@ class ApplicationAttemptInfo private[spark](
val lastUpdated: Date,
val duration: Long,
val sparkUser: String,
- val completed: Boolean = false) {
+ val completed: Boolean = false,
+ val appSparkVersion: String) {
def getStartTimeEpoch: Long = startTime.getTime
def getEndTimeEpoch: Long = endTime.getTime
def getLastUpdatedEpoch: Long = lastUpdated.getTime
diff --git a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
index bf4cf79e9faa3840b2cdb97bdf3af9a451a66d30..f271c56021e951b901b9f0106bfd733722a2dbe8 100644
--- a/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
+++ b/core/src/main/scala/org/apache/spark/ui/SparkUI.scala
@@ -60,6 +60,8 @@ private[spark] class SparkUI private (
var appId: String = _
+ var appSparkVersion = org.apache.spark.SPARK_VERSION
+
private var streamingJobProgressListener: Option[SparkListener] = None
/** Initialize all components of the server. */
@@ -118,7 +120,8 @@ private[spark] class SparkUI private (
duration = 0,
lastUpdated = new Date(startTime),
sparkUser = getSparkUser,
- completed = false
+ completed = false,
+ appSparkVersion = appSparkVersion
))
))
}
@@ -139,6 +142,7 @@ private[spark] abstract class SparkUITab(parent: SparkUI, prefix: String)
def appName: String = parent.appName
+ def appSparkVersion: String = parent.appSparkVersion
}
private[spark] object SparkUI {
diff --git a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
index 79b0d81af52b54474bd480879ec0ae23483a425a..8e1aafa448bc4ec41081402c7aec8aebcaad5c40 100644
--- a/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
+++ b/core/src/main/scala/org/apache/spark/ui/UIUtils.scala
@@ -228,7 +228,7 @@ private[spark] object UIUtils extends Logging {
<div class="brand">
<a href={prependBaseUri("/")} class="brand">
<img src={prependBaseUri("/static/spark-logo-77x50px-hd.png")} />
- <span class="version">{org.apache.spark.SPARK_VERSION}</span>
+ <span class="version">{activeTab.appSparkVersion}</span>
</a>
</div>
<p class="navbar-text pull-right">
diff --git a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
index 10902ab5a83270584abd92fd3293fb7a74cb9306..f2c3ec5da88919738ce34bd0261259d68b2a3bd0 100644
--- a/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/application_list_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 10671,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479335620587,
"startTimeEpoch" : 1479335609916,
"lastUpdatedEpoch" : 0
@@ -22,6 +23,7 @@
"duration" : 101795,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479252138874,
"startTimeEpoch" : 1479252037079,
"lastUpdatedEpoch" : 0
@@ -36,6 +38,7 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917391398,
"startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
@@ -51,6 +54,7 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917380950,
"startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
@@ -62,6 +66,7 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917380890,
"startTimeEpoch" : 1430917380880,
"lastUpdatedEpoch" : 0
@@ -77,6 +82,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426633945177,
"startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
@@ -88,6 +94,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426533945177,
"startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
@@ -102,6 +109,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1425081766912,
"startTimeEpoch" : 1425081758277,
"lastUpdatedEpoch" : 0
@@ -116,6 +124,7 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981788731,
"startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
@@ -130,6 +139,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981766912,
"startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
index 10902ab5a83270584abd92fd3293fb7a74cb9306..c925c1dd8a4d38c05644831aae48c529e49b72cd 100644
--- a/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/completed_app_list_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 10671,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479335620587,
"startTimeEpoch" : 1479335609916,
"lastUpdatedEpoch" : 0
@@ -22,6 +23,7 @@
"duration" : 101795,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479252138874,
"startTimeEpoch" : 1479252037079,
"lastUpdatedEpoch" : 0
@@ -36,6 +38,7 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917391398,
"startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
@@ -51,6 +54,7 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917380950,
"startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
@@ -62,6 +66,7 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917380890,
"startTimeEpoch" : 1430917380880,
"lastUpdatedEpoch" : 0
@@ -77,6 +82,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426633945177,
"startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
@@ -88,6 +94,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426533945177,
"startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
@@ -102,6 +109,8 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
+ "appSparkVersion" : "",
"endTimeEpoch" : 1425081766912,
"startTimeEpoch" : 1425081758277,
"lastUpdatedEpoch" : 0
@@ -116,6 +125,7 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981788731,
"startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
@@ -130,6 +140,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981766912,
"startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
index 8820c717f85d99633e38ae62c207f5e231f8ae09..cc0b2b0022bd300dc518f78ecd28f32d76f7ff32 100644
--- a/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/limit_app_list_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 10671,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479335620587,
"startTimeEpoch" : 1479335609916,
"lastUpdatedEpoch" : 0
@@ -22,6 +23,7 @@
"duration" : 101795,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479252138874,
"startTimeEpoch" : 1479252037079,
"lastUpdatedEpoch" : 0
@@ -36,6 +38,7 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917391398,
"startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
index c3fe4db222ae6875f453a50f17fe8d5900fca548..fa12413eeb0e64c9b2cb6720431ae7135375b993 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate2_app_list_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981766912,
"startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
index 8281fa75aa0db0fe842baf6624cd6c397f7a1f90..a0d4a0d1c4554b9b2ccb36d1de2e8aa175bdbcd2 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxDate_app_list_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981788731,
"startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
@@ -22,6 +23,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981766912,
"startTimeEpoch" : 1422981758277,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json
index 1842f1888b78c07f51f72ea0f155c192741ec577..dfa90010c6ca15c9f946ea3d4f422e6d0d58fb19 100644
--- a/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/maxEndDate_app_list_json_expectation.json
@@ -9,6 +9,7 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917380950
@@ -20,6 +21,7 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380880,
"endTimeEpoch" : 1430917380890
@@ -35,6 +37,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1426633910242,
"endTimeEpoch" : 1426633945177
@@ -46,6 +49,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1426533910242,
"endTimeEpoch" : 1426533945177
@@ -60,6 +64,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1425081758277,
"endTimeEpoch" : 1425081766912
@@ -74,6 +79,7 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1422981779720,
"endTimeEpoch" : 1422981788731
@@ -88,6 +94,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1422981758277,
"endTimeEpoch" : 1422981766912
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json
index 24f9f21ec6501ef2b603ff8f2459ede88e4bc5ac..3ebe60e2cd0334a752c609f17f8696654b116743 100644
--- a/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_and_maxEndDate_app_list_json_expectation.json
@@ -9,6 +9,7 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917380950
@@ -20,6 +21,7 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380880,
"endTimeEpoch" : 1430917380890
@@ -35,6 +37,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1426633910242,
"endTimeEpoch" : 1426633945177
@@ -46,6 +49,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1426533910242,
"endTimeEpoch" : 1426533945177
diff --git a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
index 1930281f1a3ec46b68ee7345112b1c7e3b0c58d9..5af50abd85330b4f414eb42be9092c7414986db3 100644
--- a/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minDate_app_list_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 10671,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479335620587,
"startTimeEpoch" : 1479335609916,
"lastUpdatedEpoch" : 0
@@ -22,6 +23,7 @@
"duration" : 101795,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"endTimeEpoch" : 1479252138874,
"startTimeEpoch" : 1479252037079,
"lastUpdatedEpoch" : 0
@@ -36,6 +38,7 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917391398,
"startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
@@ -51,6 +54,7 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917380950,
"startTimeEpoch" : 1430917380893,
"lastUpdatedEpoch" : 0
@@ -62,6 +66,7 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"endTimeEpoch" : 1430917380890,
"startTimeEpoch" : 1430917380880,
"lastUpdatedEpoch" : 0
@@ -77,6 +82,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426633945177,
"startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
@@ -88,6 +94,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426533945177,
"startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
@@ -102,6 +109,7 @@
"duration" : 8635,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1425081766912,
"startTimeEpoch" : 1425081758277,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json
index 3745e8a09a98b22a3651115122f6b01c420ec9bc..74a7b40a592721a047d1215445eebcc635829f46 100644
--- a/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minEndDate_and_maxEndDate_app_list_json_expectation.json
@@ -9,6 +9,7 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917380950
@@ -20,6 +21,7 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380880,
"endTimeEpoch" : 1430917380890
@@ -35,6 +37,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1426633910242,
"endTimeEpoch" : 1426633945177
@@ -46,6 +49,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1426533910242,
"endTimeEpoch" : 1426533945177
diff --git a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
index 05233db441eda6ed828b525763933a3681401060..7f896c74b5be1960c34884787198dd5cea6a496c 100644
--- a/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/minEndDate_app_list_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 10671,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"startTimeEpoch" : 1479335609916,
"lastUpdatedEpoch" : 0,
"endTimeEpoch" : 1479335620587
@@ -22,6 +23,7 @@
"duration" : 101795,
"sparkUser" : "jose",
"completed" : true,
+ "appSparkVersion" : "2.1.0-SNAPSHOT",
"startTimeEpoch" : 1479252037079,
"lastUpdatedEpoch" : 0,
"endTimeEpoch" : 1479252138874
@@ -36,6 +38,7 @@
"duration" : 10505,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917391398
@@ -51,6 +54,7 @@
"duration" : 57,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380893,
"endTimeEpoch" : 1430917380950
@@ -62,7 +66,7 @@
"duration" : 10,
"sparkUser" : "irashid",
"completed" : true,
- "completed" : true,
+ "appSparkVersion" : "1.4.0-SNAPSHOT",
"lastUpdatedEpoch" : 0,
"startTimeEpoch" : 1430917380880,
"endTimeEpoch" : 1430917380890
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
index e8ed96dc85f095759ef05431e46c280e0c157f87..24ec6a163fc2c0ddf652a3f50e6912b77e2db5e0 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_json_expectation.json
@@ -8,6 +8,7 @@
"duration" : 9011,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1422981788731,
"startTimeEpoch" : 1422981779720,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
index 88c601512d79ff44bbc4eee9aacff6d215ea1db1..94b6d6dba76e9cf496d92a83f510d3d712e713ba 100644
--- a/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
+++ b/core/src/test/resources/HistoryServerExpectations/one_app_multi_attempt_json_expectation.json
@@ -9,6 +9,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426633945177,
"startTimeEpoch" : 1426633910242,
"lastUpdatedEpoch" : 0
@@ -20,6 +21,7 @@
"duration" : 34935,
"sparkUser" : "irashid",
"completed" : true,
+ "appSparkVersion" : "",
"endTimeEpoch" : 1426533945177,
"startTimeEpoch" : 1426533910242,
"lastUpdatedEpoch" : 0
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
index 7998e3702c12213aa33a69a3397af10f3724dc8e..871c87415d35db6cd11c2fe6c0b05155c8c5be80 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/ApplicationCacheSuite.scala
@@ -177,7 +177,7 @@ class ApplicationCacheSuite extends SparkFunSuite with Logging with MockitoSugar
ended: Long): SparkUI = {
val info = new ApplicationInfo(name, name, Some(1), Some(1), Some(1), Some(64),
Seq(new AttemptInfo(attemptId, new Date(started), new Date(ended),
- new Date(ended), ended - started, "user", completed)))
+ new Date(ended), ended - started, "user", completed, org.apache.spark.SPARK_VERSION)))
val ui = mock[SparkUI]
when(ui.getApplicationInfoList).thenReturn(List(info).iterator)
when(ui.getAppName).thenReturn(name)
diff --git a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
index 456158d41b93f5d8a67ac5fb11a5170e8332ecb5..9b3e4ec7938255e42ed7d2e1bceaedd748c45a0f 100644
--- a/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/history/FsHistoryProviderSuite.scala
@@ -109,7 +109,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc
user: String,
completed: Boolean): ApplicationHistoryInfo = {
ApplicationHistoryInfo(id, name,
- List(ApplicationAttemptInfo(None, start, end, lastMod, user, completed)))
+ List(ApplicationAttemptInfo(None, start, end, lastMod, user, completed, "")))
}
// For completed files, lastUpdated would be lastModified time.
@@ -606,7 +606,7 @@ class FsHistoryProviderSuite extends SparkFunSuite with BeforeAndAfter with Matc
if (isNewFormat) {
val newFormatStream = new FileOutputStream(file)
Utils.tryWithSafeFinally {
- EventLoggingListener.initEventLog(newFormatStream)
+ EventLoggingListener.initEventLog(newFormatStream, false, null)
} {
newFormatStream.close()
}
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index d8b37aebb5d1d42a6218b80a3ee36a639ca6bc5b..2dff1549674286da480f4524d3758eba68ddcc0a 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -42,6 +42,9 @@ object MimaExcludes {
// Exclude rules for 2.2.x
lazy val v22excludes = v21excludes ++ Seq(
+ // [SPARK-20355] Add per application spark version on the history server headerpage
+ ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.status.api.v1.ApplicationAttemptInfo.this"),
+
// [SPARK-19652][UI] Do auth checks for REST API access.
ProblemFilters.exclude[DirectMissingMethodProblem]("org.apache.spark.deploy.history.HistoryServer.withSparkUI"),
ProblemFilters.exclude[IncompatibleTemplateDefProblem]("org.apache.spark.status.api.v1.UIRootFromServletContext"),