From b6c797b08cbd08d7aab59ad0106af0f5f41ef186 Mon Sep 17 00:00:00 2001
From: Ashwin Shankar <ashankar@netflix.com>
Date: Fri, 8 May 2015 17:51:00 -0700
Subject: [PATCH] [SPARK-7451] [YARN] Preemption of executors is counted as
failure causing Spark job to fail
Added a check to handle container exit status for the preemption scenario, log an INFO message in such cases and move on.
andrewor14
Author: Ashwin Shankar <ashankar@netflix.com>
Closes #5993 from ashwinshankar77/SPARK-7451 and squashes the following commits:
90900cf [Ashwin Shankar] Fix log info message
cf8b6cf [Ashwin Shankar] Stop counting preemption of executors as failure
---
.../scala/org/apache/spark/deploy/yarn/YarnAllocator.scala | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
index b8f42dadcb..88d68d5556 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/YarnAllocator.scala
@@ -373,7 +373,9 @@ private[yarn] class YarnAllocator(
// Hadoop 2.2.X added a ContainerExitStatus we should switch to use
// there are some exit status' we shouldn't necessarily count against us, but for
// now I think its ok as none of the containers are expected to exit
- if (completedContainer.getExitStatus == -103) { // vmem limit exceeded
+ if (completedContainer.getExitStatus == ContainerExitStatus.PREEMPTED) {
+ logInfo("Container preempted: " + containerId)
+ } else if (completedContainer.getExitStatus == -103) { // vmem limit exceeded
logWarning(memLimitExceededLogMessage(
completedContainer.getDiagnostics,
VMEM_EXCEEDED_PATTERN))
--
GitLab