diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md index 62b317129b72a9407225e479803bf7b18e04f5e5..b5fb077441468458eb121c7db642568f3c107464 100644 --- a/docs/running-on-yarn.md +++ b/docs/running-on-yarn.md @@ -139,6 +139,14 @@ Most of the configs are the same for Spark on YARN as for other deployment modes The maximum number of threads to use in the application master for launching executor containers. </td> </tr> +<tr> + <td><code>spark.yarn.am.extraJavaOptions</code></td> + <td>(none)</td> + <td> + A string of extra JVM options to pass to the Yarn ApplicationMaster in client mode. + In cluster mode, use spark.driver.extraJavaOptions instead. + </td> +</tr> </table> # Launching Spark on YARN diff --git a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala index f95d72379171c3997bd74d15cb9473e086764892..5f0c67f05c9ddc97da16a8b410e792f26f7f001e 100644 --- a/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala +++ b/yarn/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala @@ -352,12 +352,31 @@ private[spark] trait ClientBase extends Logging { if (isLaunchingDriver) { sparkConf.getOption("spark.driver.extraJavaOptions") .orElse(sys.env.get("SPARK_JAVA_OPTS")) + .map(Utils.splitCommandString).getOrElse(Seq.empty) .foreach(opts => javaOpts += opts) val libraryPaths = Seq(sys.props.get("spark.driver.extraLibraryPath"), sys.props.get("spark.driver.libraryPath")).flatten if (libraryPaths.nonEmpty) { prefixEnv = Some(Utils.libraryPathEnvPrefix(libraryPaths)) } + if (sparkConf.getOption("spark.yarn.am.extraJavaOptions").isDefined) { + logWarning("spark.yarn.am.extraJavaOptions will not take effect in cluster mode") + } + } else { + // Validate and include yarn am specific java options in yarn-client mode. + val amOptsKey = "spark.yarn.am.extraJavaOptions" + val amOpts = sparkConf.getOption(amOptsKey) + amOpts.foreach { opts => + if (opts.contains("-Dspark")) { + val msg = s"$amOptsKey is not allowed to set Spark options (was '$opts'). " + throw new SparkException(msg) + } + if (opts.contains("-Xmx") || opts.contains("-Xms")) { + val msg = s"$amOptsKey is not allowed to alter memory settings (was '$opts')." + throw new SparkException(msg) + } + javaOpts ++= Utils.splitCommandString(opts) + } } // For log4j configuration to reference