Skip to content
Snippets Groups Projects
Commit 8b784129 authored by Andrew Or's avatar Andrew Or Committed by Patrick Wendell
Browse files

[SPARK-1755] Respect SparkSubmit --name on YARN

Right now, SparkSubmit ignores the `--name` flag for both yarn-client and yarn-cluster. This is a bug.

In client mode, SparkSubmit treats `--name` as a [cluster config](https://github.com/apache/spark/blob/master/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala#L170) and does not propagate this to SparkContext.

In cluster mode, SparkSubmit passes this flag to `org.apache.spark.deploy.yarn.Client`, which only uses it for the [YARN ResourceManager](https://github.com/apache/spark/blob/master/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala#L80), but does not propagate this to SparkContext.

This PR ensures that `spark.app.name` is always set if SparkSubmit receives the `--name` flag, which is what the usage promises. This makes it possible for applications to start a SparkContext with an empty conf `val sc = new SparkContext(new SparkConf)`, and inherit the app name from SparkSubmit.

Tested both modes on a YARN cluster.

Author: Andrew Or <andrewor14@gmail.com>

Closes #699 from andrewor14/yarn-app-name and squashes the following commits:

98f6a79 [Andrew Or] Fix tests
dea932f [Andrew Or] Merge branch 'master' of github.com:apache/spark into yarn-app-name
c86d9ca [Andrew Or] Respect SparkSubmit --name on YARN
parent 2fd2752e
No related branches found
No related tags found
No related merge requests found
...@@ -160,6 +160,7 @@ object SparkSubmit { ...@@ -160,6 +160,7 @@ object SparkSubmit {
// each deploy mode; we iterate through these below // each deploy mode; we iterate through these below
val options = List[OptionAssigner]( val options = List[OptionAssigner](
OptionAssigner(args.master, ALL_CLUSTER_MGRS, false, sysProp = "spark.master"), OptionAssigner(args.master, ALL_CLUSTER_MGRS, false, sysProp = "spark.master"),
OptionAssigner(args.name, ALL_CLUSTER_MGRS, false, sysProp = "spark.app.name"),
OptionAssigner(args.driverExtraClassPath, STANDALONE | YARN, true, OptionAssigner(args.driverExtraClassPath, STANDALONE | YARN, true,
sysProp = "spark.driver.extraClassPath"), sysProp = "spark.driver.extraClassPath"),
OptionAssigner(args.driverExtraJavaOptions, STANDALONE | YARN, true, OptionAssigner(args.driverExtraJavaOptions, STANDALONE | YARN, true,
...@@ -167,7 +168,7 @@ object SparkSubmit { ...@@ -167,7 +168,7 @@ object SparkSubmit {
OptionAssigner(args.driverExtraLibraryPath, STANDALONE | YARN, true, OptionAssigner(args.driverExtraLibraryPath, STANDALONE | YARN, true,
sysProp = "spark.driver.extraLibraryPath"), sysProp = "spark.driver.extraLibraryPath"),
OptionAssigner(args.driverMemory, YARN, true, clOption = "--driver-memory"), OptionAssigner(args.driverMemory, YARN, true, clOption = "--driver-memory"),
OptionAssigner(args.name, YARN, true, clOption = "--name"), OptionAssigner(args.name, YARN, true, clOption = "--name", sysProp = "spark.app.name"),
OptionAssigner(args.queue, YARN, true, clOption = "--queue"), OptionAssigner(args.queue, YARN, true, clOption = "--queue"),
OptionAssigner(args.queue, YARN, false, sysProp = "spark.yarn.queue"), OptionAssigner(args.queue, YARN, false, sysProp = "spark.yarn.queue"),
OptionAssigner(args.numExecutors, YARN, true, clOption = "--num-executors"), OptionAssigner(args.numExecutors, YARN, true, clOption = "--num-executors"),
...@@ -188,8 +189,7 @@ object SparkSubmit { ...@@ -188,8 +189,7 @@ object SparkSubmit {
OptionAssigner(args.jars, YARN, true, clOption = "--addJars"), OptionAssigner(args.jars, YARN, true, clOption = "--addJars"),
OptionAssigner(args.files, LOCAL | STANDALONE | MESOS, false, sysProp = "spark.files"), OptionAssigner(args.files, LOCAL | STANDALONE | MESOS, false, sysProp = "spark.files"),
OptionAssigner(args.files, LOCAL | STANDALONE | MESOS, true, sysProp = "spark.files"), OptionAssigner(args.files, LOCAL | STANDALONE | MESOS, true, sysProp = "spark.files"),
OptionAssigner(args.jars, LOCAL | STANDALONE | MESOS, false, sysProp = "spark.jars"), OptionAssigner(args.jars, LOCAL | STANDALONE | MESOS, false, sysProp = "spark.jars")
OptionAssigner(args.name, LOCAL | STANDALONE | MESOS, false, sysProp = "spark.app.name")
) )
// For client mode make any added jars immediately visible on the classpath // For client mode make any added jars immediately visible on the classpath
...@@ -205,7 +205,8 @@ object SparkSubmit { ...@@ -205,7 +205,8 @@ object SparkSubmit {
(clusterManager & opt.clusterManager) != 0) { (clusterManager & opt.clusterManager) != 0) {
if (opt.clOption != null) { if (opt.clOption != null) {
childArgs += (opt.clOption, opt.value) childArgs += (opt.clOption, opt.value)
} else if (opt.sysProp != null) { }
if (opt.sysProp != null) {
sysProps.put(opt.sysProp, opt.value) sysProps.put(opt.sysProp, opt.value)
} }
} }
......
...@@ -104,7 +104,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers { ...@@ -104,7 +104,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers {
"--master", "yarn", "--executor-memory", "5g", "--executor-cores", "5", "--master", "yarn", "--executor-memory", "5g", "--executor-cores", "5",
"--class", "org.SomeClass", "--jars", "one.jar,two.jar,three.jar", "--class", "org.SomeClass", "--jars", "one.jar,two.jar,three.jar",
"--driver-memory", "4g", "--queue", "thequeue", "--files", "file1.txt,file2.txt", "--driver-memory", "4g", "--queue", "thequeue", "--files", "file1.txt,file2.txt",
"--archives", "archive1.txt,archive2.txt", "--num-executors", "6", "--archives", "archive1.txt,archive2.txt", "--num-executors", "6", "--name", "beauty",
"thejar.jar", "arg1", "arg2") "thejar.jar", "arg1", "arg2")
val appArgs = new SparkSubmitArguments(clArgs) val appArgs = new SparkSubmitArguments(clArgs)
val (childArgs, classpath, sysProps, mainClass) = createLaunchEnv(appArgs) val (childArgs, classpath, sysProps, mainClass) = createLaunchEnv(appArgs)
...@@ -122,7 +122,8 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers { ...@@ -122,7 +122,8 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers {
childArgsStr should include ("--num-executors 6") childArgsStr should include ("--num-executors 6")
mainClass should be ("org.apache.spark.deploy.yarn.Client") mainClass should be ("org.apache.spark.deploy.yarn.Client")
classpath should have length (0) classpath should have length (0)
sysProps should have size (1) sysProps("spark.app.name") should be ("beauty")
sysProps("SPARK_SUBMIT") should be ("true")
} }
test("handles YARN client mode") { test("handles YARN client mode") {
...@@ -130,8 +131,8 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers { ...@@ -130,8 +131,8 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers {
"--master", "yarn", "--executor-memory", "5g", "--executor-cores", "5", "--master", "yarn", "--executor-memory", "5g", "--executor-cores", "5",
"--class", "org.SomeClass", "--jars", "one.jar,two.jar,three.jar", "--class", "org.SomeClass", "--jars", "one.jar,two.jar,three.jar",
"--driver-memory", "4g", "--queue", "thequeue", "--files", "file1.txt,file2.txt", "--driver-memory", "4g", "--queue", "thequeue", "--files", "file1.txt,file2.txt",
"--archives", "archive1.txt,archive2.txt", "--num-executors", "6", "thejar.jar", "--archives", "archive1.txt,archive2.txt", "--num-executors", "6", "--name", "trill",
"arg1", "arg2") "thejar.jar", "arg1", "arg2")
val appArgs = new SparkSubmitArguments(clArgs) val appArgs = new SparkSubmitArguments(clArgs)
val (childArgs, classpath, sysProps, mainClass) = createLaunchEnv(appArgs) val (childArgs, classpath, sysProps, mainClass) = createLaunchEnv(appArgs)
childArgs.mkString(" ") should be ("arg1 arg2") childArgs.mkString(" ") should be ("arg1 arg2")
...@@ -140,6 +141,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers { ...@@ -140,6 +141,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers {
classpath should contain ("one.jar") classpath should contain ("one.jar")
classpath should contain ("two.jar") classpath should contain ("two.jar")
classpath should contain ("three.jar") classpath should contain ("three.jar")
sysProps("spark.app.name") should be ("trill")
sysProps("spark.executor.memory") should be ("5g") sysProps("spark.executor.memory") should be ("5g")
sysProps("spark.executor.cores") should be ("5") sysProps("spark.executor.cores") should be ("5")
sysProps("spark.yarn.queue") should be ("thequeue") sysProps("spark.yarn.queue") should be ("thequeue")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment