From 80098109d908b738b43d397e024756ff617d0af4 Mon Sep 17 00:00:00 2001
From: "Zhang, Liye" <liye.zhang@intel.com>
Date: Tue, 28 Apr 2015 12:33:48 -0700
Subject: [PATCH] [SPARK-6314] [CORE] handle JsonParseException for history
server
This is handled in the same way with [SPARK-6197](https://issues.apache.org/jira/browse/SPARK-6197). The result of this PR is that exception showed in history server log will be replaced by a warning, and the application that with un-complete history log file will be listed on history server webUI
Author: Zhang, Liye <liye.zhang@intel.com>
Closes #5736 from liyezhang556520/SPARK-6314 and squashes the following commits:
b8d2d88 [Zhang, Liye] handle JsonParseException for history server
---
.../org/apache/spark/deploy/history/FsHistoryProvider.scala | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
index a94ebf6e53..fb2cbbcccc 100644
--- a/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/history/FsHistoryProvider.scala
@@ -333,8 +333,9 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis
}
try {
val appListener = new ApplicationEventListener
+ val appCompleted = isApplicationCompleted(eventLog)
bus.addListener(appListener)
- bus.replay(logInput, logPath.toString)
+ bus.replay(logInput, logPath.toString, !appCompleted)
new FsApplicationHistoryInfo(
logPath.getName(),
appListener.appId.getOrElse(logPath.getName()),
@@ -343,7 +344,7 @@ private[history] class FsHistoryProvider(conf: SparkConf) extends ApplicationHis
appListener.endTime.getOrElse(-1L),
getModificationTime(eventLog).get,
appListener.sparkUser.getOrElse(NOT_STARTED),
- isApplicationCompleted(eventLog))
+ appCompleted)
} finally {
logInput.close()
}
--
GitLab