From ca0690b5ef10b14ce57a0c30d5308eb02f163f39 Mon Sep 17 00:00:00 2001
From: Devaraj K <devaraj@apache.org>
Date: Tue, 15 Dec 2015 18:30:59 -0800
Subject: [PATCH] [SPARK-4117][YARN] Spark on Yarn handle AM being told command
from RM
Spark on Yarn handle AM being told command from RM
When RM throws ApplicationAttemptNotFoundException for allocate
invocation, making the ApplicationMaster to finish immediately without any
retries.
Author: Devaraj K <devaraj@apache.org>
Closes #10129 from devaraj-kavali/SPARK-4117.
---
.../org/apache/spark/deploy/yarn/ApplicationMaster.scala | 9 ++++++++-
1 file changed, 8 insertions(+), 1 deletion(-)
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
index 1970f7d150..fc742df73d 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ApplicationMaster.scala
@@ -376,7 +376,14 @@ private[spark] class ApplicationMaster(
case i: InterruptedException =>
case e: Throwable => {
failureCount += 1
- if (!NonFatal(e) || failureCount >= reporterMaxFailures) {
+ // this exception was introduced in hadoop 2.4 and this code would not compile
+ // with earlier versions if we refer it directly.
+ if ("org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException" ==
+ e.getClass().getName()) {
+ logError("Exception from Reporter thread.", e)
+ finish(FinalApplicationStatus.FAILED, ApplicationMaster.EXIT_REPORTER_FAILURE,
+ e.getMessage)
+ } else if (!NonFatal(e) || failureCount >= reporterMaxFailures) {
finish(FinalApplicationStatus.FAILED,
ApplicationMaster.EXIT_REPORTER_FAILURE, "Exception was thrown " +
s"$failureCount time(s) from Reporter thread.")
--
GitLab