From 06fb2d057beb50e9b690bf8b6d5bb7bdb16d8546 Mon Sep 17 00:00:00 2001
From: Chris Cope <ccope@resilientscience.com>
Date: Thu, 11 Sep 2014 08:13:07 -0500
Subject: [PATCH] [SPARK-2140] Updating heap memory calculation for YARN stable
and alpha.
Updated pull request, reflecting YARN stable and alpha states. I am getting intermittent test failures on my own test infrastructure. Is that tracked anywhere yet?
Author: Chris Cope <ccope@resilientscience.com>
Closes #2253 from copester/master and squashes the following commits:
5ad89da [Chris Cope] [SPARK-2140] Removing calculateAMMemory functions since they are no longer needed.
52b4e45 [Chris Cope] [SPARK-2140] Updating heap memory calculation for YARN stable and alpha.
(cherry picked from commit ed1980ffa9ccb87d76694ba910ef22df034bca49)
Signed-off-by: Thomas Graves <tgraves@apache.org>
---
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 8 --------
.../scala/org/apache/spark/deploy/yarn/ClientBase.scala | 4 +---
.../org/apache/spark/deploy/yarn/ClientBaseSuite.scala | 3 ---
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 9 ---------
4 files changed, 1 insertion(+), 23 deletions(-)
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 9be78546c1..3607eed1f1 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -102,14 +102,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
appContext
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int = {
- val minResMemory = newApp.getMinimumResourceCapability().getMemory()
- val amMemory = ((args.amMemory / minResMemory) * minResMemory) +
- ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
- memoryOverhead)
- amMemory
- }
-
def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index 3897b3a373..6da3b16a54 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -309,8 +309,6 @@ trait ClientBase extends Logging {
retval.toString
}
- def calculateAMMemory(newApp: GetNewApplicationResponse): Int
-
def setupSecurityToken(amContainer: ContainerLaunchContext)
def createContainerLaunchContext(
@@ -353,7 +351,7 @@ trait ClientBase extends Logging {
}
amContainer.setEnvironment(env)
- val amMemory = calculateAMMemory(newApp)
+ val amMemory = args.amMemory
val javaOpts = ListBuffer[String]()
diff --git a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
index 68cc2890f3..5480eca7c8 100644
--- a/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
+++ b/yarn/common/src/test/scala/org/apache/spark/deploy/yarn/ClientBaseSuite.scala
@@ -238,9 +238,6 @@ class ClientBaseSuite extends FunSuite with Matchers {
val sparkConf: SparkConf,
val yarnConf: YarnConfiguration) extends ClientBase {
- override def calculateAMMemory(newApp: GetNewApplicationResponse): Int =
- throw new UnsupportedOperationException()
-
override def setupSecurityToken(amContainer: ContainerLaunchContext): Unit =
throw new UnsupportedOperationException()
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 1f9a4bf209..0b55282b75 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -103,15 +103,6 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
clusterMetrics.getNumNodeManagers)
}
- def calculateAMMemory(newApp: GetNewApplicationResponse) :Int = {
- // TODO: Need a replacement for the following code to fix -Xmx?
- // val minResMemory: Int = newApp.getMinimumResourceCapability().getMemory()
- // var amMemory = ((args.amMemory / minResMemory) * minResMemory) +
- // ((if ((args.amMemory % minResMemory) == 0) 0 else minResMemory) -
- // memoryOverhead )
- args.amMemory
- }
-
def setupSecurityToken(amContainer: ContainerLaunchContext) = {
// Setup security tokens.
val dob = new DataOutputBuffer()
--
GitLab