diff --git a/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala b/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
index 0c096656f9236daccbb70b288f3317d1f856631c..a2add61617281a604f2db9d8a90bc4404ceaf513 100644
--- a/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
+++ b/core/src/main/scala/org/apache/spark/launcher/WorkerCommandBuilder.scala
@@ -40,7 +40,7 @@ private[spark] class WorkerCommandBuilder(sparkHome: String, memoryMb: Int, comm
cmd.add(s"-Xms${memoryMb}M")
cmd.add(s"-Xmx${memoryMb}M")
command.javaOpts.foreach(cmd.add)
- addPermGenSizeOpt(cmd)
+ CommandBuilderUtils.addPermGenSizeOpt(cmd)
addOptionString(cmd, getenv("SPARK_JAVA_OPTS"))
cmd
}
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index 0a237ee73b670ec21a7d7815ee880125d791e854..610e8bdaaa63998045fe7b76fce7aa8f7ccfefab 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -116,29 +116,6 @@ abstract class AbstractCommandBuilder {
return cmd;
}
- /**
- * Adds the default perm gen size option for Spark if the VM requires it and the user hasn't
- * set it.
- */
- void addPermGenSizeOpt(List<String> cmd) {
- // Don't set MaxPermSize for IBM Java, or Oracle Java 8 and later.
- if (getJavaVendor() == JavaVendor.IBM) {
- return;
- }
- String[] version = System.getProperty("java.version").split("\\.");
- if (Integer.parseInt(version[0]) > 1 || Integer.parseInt(version[1]) > 7) {
- return;
- }
-
- for (String arg : cmd) {
- if (arg.startsWith("-XX:MaxPermSize=")) {
- return;
- }
- }
-
- cmd.add("-XX:MaxPermSize=256m");
- }
-
void addOptionString(List<String> cmd, String options) {
if (!isEmpty(options)) {
for (String opt : parseOptionString(options)) {
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index a16c0d2b5ca0b9a4871815c73b8e3e799727a2e5..d30c2ec5f87bbded5ad187b33c4f90182becaa0a 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -313,4 +313,27 @@ class CommandBuilderUtils {
return quoted.append('"').toString();
}
+ /**
+ * Adds the default perm gen size option for Spark if the VM requires it and the user hasn't
+ * set it.
+ */
+ static void addPermGenSizeOpt(List<String> cmd) {
+ // Don't set MaxPermSize for IBM Java, or Oracle Java 8 and later.
+ if (getJavaVendor() == JavaVendor.IBM) {
+ return;
+ }
+ String[] version = System.getProperty("java.version").split("\\.");
+ if (Integer.parseInt(version[0]) > 1 || Integer.parseInt(version[1]) > 7) {
+ return;
+ }
+
+ for (String arg : cmd) {
+ if (arg.startsWith("-XX:MaxPermSize=")) {
+ return;
+ }
+ }
+
+ cmd.add("-XX:MaxPermSize=256m");
+ }
+
}
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 8c53c24a79c4815869b317bc09b7eb049202bdca..f8748ef6587af4f23c87d615319396a1f077c975 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -54,8 +54,9 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException
import org.apache.hadoop.yarn.util.Records
-import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkContext, SparkException}
+import org.apache.spark.deploy.SparkHadoopUtil
+import org.apache.spark.launcher.YarnCommandBuilderUtils
import org.apache.spark.util.Utils
private[spark] class Client(
@@ -730,6 +731,7 @@ private[spark] class Client(
// For log4j configuration to reference
javaOpts += ("-Dspark.yarn.app.container.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR)
+ YarnCommandBuilderUtils.addPermGenSizeOpt(javaOpts)
val userClass =
if (isClusterMode) {
diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
index 9abd09b3cc7a52bd76ff58a347a93aeeab6a4cff..2232ffba473b571aa74c00b16aa13b15cbeed80f 100644
--- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
+++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnable.scala
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.ipc.YarnRPC
import org.apache.hadoop.yarn.util.{ConverterUtils, Records}
import org.apache.spark.{Logging, SecurityManager, SparkConf, SparkException}
+import org.apache.spark.launcher.YarnCommandBuilderUtils
import org.apache.spark.network.util.JavaUtils
import org.apache.spark.util.Utils
@@ -199,6 +200,7 @@ class ExecutorRunnable(
// For log4j configuration to reference
javaOpts += ("-Dspark.yarn.app.container.log.dir=" + ApplicationConstants.LOG_DIR_EXPANSION_VAR)
+ YarnCommandBuilderUtils.addPermGenSizeOpt(javaOpts)
val userClassPath = Client.getUserClasspath(sparkConf).flatMap { uri =>
val absPath =
diff --git a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
index 3ac36ef0a1c3f158b8c297df75d14792c89a154e..7d246bf40712182cd7e18d81b164b9fec53bfa3b 100644
--- a/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
+++ b/yarn/src/main/scala/org/apache/spark/launcher/YarnCommandBuilderUtils.scala
@@ -17,11 +17,28 @@
package org.apache.spark.launcher
+import scala.collection.JavaConverters._
+import scala.collection.mutable.ListBuffer
+
/**
- * Exposes needed methods
+ * Exposes methods from the launcher library that are used by the YARN backend.
*/
private[spark] object YarnCommandBuilderUtils {
- def quoteForBatchScript(arg: String) : String = {
+
+ def quoteForBatchScript(arg: String): String = {
CommandBuilderUtils.quoteForBatchScript(arg)
}
+
+ /**
+ * Adds the perm gen configuration to the list of java options if needed and not yet added.
+ *
+ * Note that this method adds the option based on the local JVM version; if the node where
+ * the container is running has a different Java version, there's a risk that the option will
+ * not be added (e.g. if the AM is running Java 8 but the container's node is set up to use
+ * Java 7).
+ */
+ def addPermGenSizeOpt(args: ListBuffer[String]): Unit = {
+ CommandBuilderUtils.addPermGenSizeOpt(args.asJava)
+ }
+
}