From 51898b5158ac7e7e67b0539bc062c9c16ce9a7ce Mon Sep 17 00:00:00 2001
From: Ryan Williams <ryan.blake.williams@gmail.com>
Date: Wed, 3 Jun 2015 16:54:46 -0700
Subject: [PATCH] [SPARK-8088] don't attempt to lower number of executors by 0
Author: Ryan Williams <ryan.blake.williams@gmail.com>
Closes #6624 from ryan-williams/execs and squashes the following commits:
b6f71d4 [Ryan Williams] don't attempt to lower number of executors by 0
---
.../org/apache/spark/ExecutorAllocationManager.scala | 10 +++++++---
1 file changed, 7 insertions(+), 3 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
index 9514604752..f7323a4d9d 100644
--- a/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
+++ b/core/src/main/scala/org/apache/spark/ExecutorAllocationManager.scala
@@ -266,10 +266,14 @@ private[spark] class ExecutorAllocationManager(
// executors and inform the cluster manager to cancel the extra pending requests
val oldNumExecutorsTarget = numExecutorsTarget
numExecutorsTarget = math.max(maxNeeded, minNumExecutors)
- client.requestTotalExecutors(numExecutorsTarget)
numExecutorsToAdd = 1
- logInfo(s"Lowering target number of executors to $numExecutorsTarget because " +
- s"not all requests are actually needed (previously $oldNumExecutorsTarget)")
+
+ // If the new target has not changed, avoid sending a message to the cluster manager
+ if (numExecutorsTarget < oldNumExecutorsTarget) {
+ client.requestTotalExecutors(numExecutorsTarget)
+ logInfo(s"Lowering target number of executors to $numExecutorsTarget (previously " +
+ s"$oldNumExecutorsTarget) because not all requested executors are actually needed")
+ }
numExecutorsTarget - oldNumExecutorsTarget
} else if (addTime != NOT_SET && now >= addTime) {
val delta = addExecutors(maxNeeded)
--
GitLab