From 297813647508480d7b4b5bccd02b93b8b914301f Mon Sep 17 00:00:00 2001
From: Sean Owen <sowen@cloudera.com>
Date: Wed, 26 Oct 2016 14:23:11 +0200
Subject: [PATCH] [SPARK-18027][YARN] .sparkStaging not clean on RM
ApplicationNotFoundException
## What changes were proposed in this pull request?
Cleanup YARN staging dir on all `KILLED`/`FAILED` paths in `monitorApplication`
## How was this patch tested?
Existing tests
Author: Sean Owen <sowen@cloudera.com>
Closes #15598 from srowen/SPARK-18027.
---
yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala | 2 ++
1 file changed, 2 insertions(+)
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 6e4f68c74c..55e4a833b6 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -1059,9 +1059,11 @@ private[spark] class Client(
} catch {
case e: ApplicationNotFoundException =>
logError(s"Application $appId not found.")
+ cleanupStagingDir(appId)
return (YarnApplicationState.KILLED, FinalApplicationStatus.KILLED)
case NonFatal(e) =>
logError(s"Failed to contact YARN for application $appId.", e)
+ // Don't necessarily clean up staging dir because status is unknown
return (YarnApplicationState.FAILED, FinalApplicationStatus.FAILED)
}
val state = report.getYarnApplicationState
--
GitLab