diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template index 1663019ee57589cc4835544cdbd37e0e09ede4ed..f8c895f5303b944c20d07d3a823f83656c3878e9 100755 --- a/conf/spark-env.sh.template +++ b/conf/spark-env.sh.template @@ -52,6 +52,7 @@ # - SPARK_HISTORY_OPTS, to set config properties only for the history server (e.g. "-Dx=y") # - SPARK_SHUFFLE_OPTS, to set config properties only for the external shuffle service (e.g. "-Dx=y") # - SPARK_DAEMON_JAVA_OPTS, to set config properties for all daemons (e.g. "-Dx=y") +# - SPARK_DAEMON_CLASSPATH, to set the classpath for all daemons # - SPARK_PUBLIC_DNS, to set the public dns name of the master or workers # Generic options for the daemons used in the standalone deploy mode diff --git a/docs/monitoring.md b/docs/monitoring.md index 3e577c5f36778512e10c8d69ffc94f3eef4ee1b5..d22cd945eaf6177c53c229c6e124316df0f0b547 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -61,6 +61,10 @@ The history server can be configured as follows: <td><code>SPARK_DAEMON_JAVA_OPTS</code></td> <td>JVM options for the history server (default: none).</td> </tr> + <tr> + <td><code>SPARK_DAEMON_CLASSPATH</code></td> + <td>Classpath for the history server (default: none).</td> + </tr> <tr> <td><code>SPARK_PUBLIC_DNS</code></td> <td> diff --git a/docs/running-on-mesos.md b/docs/running-on-mesos.md index 0e5a20c578db3ba81f9b3e5c2d9c0e71b54467f5..c12b8580af063f48fe5288df82f2b1e1c6288f52 100644 --- a/docs/running-on-mesos.md +++ b/docs/running-on-mesos.md @@ -160,6 +160,8 @@ If you like to run the `MesosClusterDispatcher` with Marathon, you need to run t The `MesosClusterDispatcher` also supports writing recovery state into Zookeeper. This will allow the `MesosClusterDispatcher` to be able to recover all submitted and running containers on relaunch. In order to enable this recovery mode, you can set SPARK_DAEMON_JAVA_OPTS in spark-env by configuring `spark.deploy.recoveryMode` and related spark.deploy.zookeeper.* configurations. For more information about these configurations please refer to the configurations [doc](configurations.html#deploy). +You can also specify any additional jars required by the `MesosClusterDispatcher` in the classpath by setting the environment variable SPARK_DAEMON_CLASSPATH in spark-env. + From the client, you can submit a job to Mesos cluster by running `spark-submit` and specifying the master URL to the URL of the `MesosClusterDispatcher` (e.g: mesos://dispatcher:7077). You can view driver statuses on the Spark cluster Web UI. diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md index 642575b46dd4282da5212710e2c57770cb27c0d4..1095386c31ab8d6377ff02da5cc6e9052eafdeba 100644 --- a/docs/spark-standalone.md +++ b/docs/spark-standalone.md @@ -149,6 +149,10 @@ You can optionally configure the cluster further by setting environment variable <td><code>SPARK_DAEMON_JAVA_OPTS</code></td> <td>JVM options for the Spark master and worker daemons themselves in the form "-Dx=y" (default: none).</td> </tr> + <tr> + <td><code>SPARK_DAEMON_CLASSPATH</code></td> + <td>Classpath for the Spark master and worker daemons themselves (default: none).</td> + </tr> <tr> <td><code>SPARK_PUBLIC_DNS</code></td> <td>The public DNS name of the Spark master and workers (default: none).</td> diff --git a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java index 137ef74843da571f0b3359cfb326e8d421f2fcbb..32724acdc362c2334ef7be93324925f12e93b660 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java +++ b/launcher/src/main/java/org/apache/spark/launcher/SparkClassCommandBuilder.java @@ -53,16 +53,19 @@ class SparkClassCommandBuilder extends AbstractCommandBuilder { case "org.apache.spark.deploy.master.Master": javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); javaOptsKeys.add("SPARK_MASTER_OPTS"); + extraClassPath = getenv("SPARK_DAEMON_CLASSPATH"); memKey = "SPARK_DAEMON_MEMORY"; break; case "org.apache.spark.deploy.worker.Worker": javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); javaOptsKeys.add("SPARK_WORKER_OPTS"); + extraClassPath = getenv("SPARK_DAEMON_CLASSPATH"); memKey = "SPARK_DAEMON_MEMORY"; break; case "org.apache.spark.deploy.history.HistoryServer": javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); javaOptsKeys.add("SPARK_HISTORY_OPTS"); + extraClassPath = getenv("SPARK_DAEMON_CLASSPATH"); memKey = "SPARK_DAEMON_MEMORY"; break; case "org.apache.spark.executor.CoarseGrainedExecutorBackend": @@ -77,11 +80,13 @@ class SparkClassCommandBuilder extends AbstractCommandBuilder { break; case "org.apache.spark.deploy.mesos.MesosClusterDispatcher": javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); + extraClassPath = getenv("SPARK_DAEMON_CLASSPATH"); break; case "org.apache.spark.deploy.ExternalShuffleService": case "org.apache.spark.deploy.mesos.MesosExternalShuffleService": javaOptsKeys.add("SPARK_DAEMON_JAVA_OPTS"); javaOptsKeys.add("SPARK_SHUFFLE_OPTS"); + extraClassPath = getenv("SPARK_DAEMON_CLASSPATH"); memKey = "SPARK_DAEMON_MEMORY"; break; default: