From f6773edce05300faf1e673ea2d1782dfb9b8b998 Mon Sep 17 00:00:00 2001
From: "Zhang, Liye" <liye.zhang@intel.com>
Date: Wed, 4 Mar 2015 12:28:27 +0000
Subject: [PATCH] [SPARK-6107][CORE] Display inprogress application information
for event log history for standalone mode
when application is finished running abnormally (Ctrl + c for example), the history event log file is still ends with `.inprogress` suffix. And the application state can not be showed on webUI, User can only see "*Application history not foud xxxx, Application xxx is still in progress*".
For application that not finished normally, the history will show:
Author: Zhang, Liye <liye.zhang@intel.com>
Closes #4848 from liyezhang556520/showLogInprogress and squashes the following commits:
03589ac [Zhang, Liye] change inprogress to in progress
b55f19f [Zhang, Liye] scala modify after rebase
8aa66a2 [Zhang, Liye] use softer wording
b030bd4 [Zhang, Liye] clean code
79c8cb1 [Zhang, Liye] fix some mistakes
11cdb68 [Zhang, Liye] add a missing space
c29205b [Zhang, Liye] refine code according to sean owen's comments
e9952a7 [Zhang, Liye] scala style fix again
150502d [Zhang, Liye] scala style fix
f11a5da [Zhang, Liye] small fix for file path
22e878b [Zhang, Liye] enable in progress eventlog file
---
.../apache/spark/deploy/master/Master.scala | 34 +++++++++++--------
1 file changed, 19 insertions(+), 15 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
index 148485cc11..4584b730e3 100644
--- a/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/master/Master.scala
@@ -736,30 +736,34 @@ private[spark] class Master(
val appName = app.desc.name
val notFoundBasePath = HistoryServer.UI_PATH_PREFIX + "/not-found"
try {
- val eventLogFile = app.desc.eventLogDir
- .map { dir => EventLoggingListener.getLogPath(dir, app.id, app.desc.eventLogCodec) }
+ val eventLogDir = app.desc.eventLogDir
.getOrElse {
// Event logging is not enabled for this application
app.desc.appUiUrl = notFoundBasePath
return false
}
-
- val fs = Utils.getHadoopFileSystem(eventLogFile, hadoopConf)
-
- if (fs.exists(new Path(eventLogFile + EventLoggingListener.IN_PROGRESS))) {
+
+ val eventLogFilePrefix = EventLoggingListener.getLogPath(
+ eventLogDir, app.id, app.desc.eventLogCodec)
+ val fs = Utils.getHadoopFileSystem(eventLogDir, hadoopConf)
+ val inProgressExists = fs.exists(new Path(eventLogFilePrefix +
+ EventLoggingListener.IN_PROGRESS))
+
+ if (inProgressExists) {
// Event logging is enabled for this application, but the application is still in progress
- val title = s"Application history not found (${app.id})"
- var msg = s"Application $appName is still in progress."
- logWarning(msg)
- msg = URLEncoder.encode(msg, "UTF-8")
- app.desc.appUiUrl = notFoundBasePath + s"?msg=$msg&title=$title"
- return false
+ logWarning(s"Application $appName is still in progress, it may be terminated abnormally.")
}
-
+
+ val (eventLogFile, status) = if (inProgressExists) {
+ (eventLogFilePrefix + EventLoggingListener.IN_PROGRESS, " (in progress)")
+ } else {
+ (eventLogFilePrefix, " (completed)")
+ }
+
val logInput = EventLoggingListener.openEventLog(new Path(eventLogFile), fs)
val replayBus = new ReplayListenerBus()
val ui = SparkUI.createHistoryUI(new SparkConf, replayBus, new SecurityManager(conf),
- appName + " (completed)", HistoryServer.UI_PATH_PREFIX + s"/${app.id}")
+ appName + status, HistoryServer.UI_PATH_PREFIX + s"/${app.id}")
try {
replayBus.replay(logInput, eventLogFile)
} finally {
@@ -774,7 +778,7 @@ private[spark] class Master(
case fnf: FileNotFoundException =>
// Event logging is enabled for this application, but no event logs are found
val title = s"Application history not found (${app.id})"
- var msg = s"No event logs found for application $appName in ${app.desc.eventLogDir}."
+ var msg = s"No event logs found for application $appName in ${app.desc.eventLogDir.get}."
logWarning(msg)
msg += " Did you specify the correct logging directory?"
msg = URLEncoder.encode(msg, "UTF-8")
--
GitLab