From dcbc4265839d8a6f0300af6bcc1e8d8c102ec23f Mon Sep 17 00:00:00 2001
From: "Susan X. Huynh" <xhuynh@mesosphere.com>
Date: Sat, 5 Nov 2016 17:45:15 +0000
Subject: [PATCH] [SPARK-17964][SPARKR] Enable SparkR with Mesos client mode
and cluster mode
## What changes were proposed in this pull request?
Enabled SparkR with Mesos client mode and cluster mode. Just a few changes were required to get this working on Mesos: (1) removed the SparkR on Mesos error checks and (2) do not require "--class" to be specified for R apps. The logic to check spark.mesos.executor.home was already in there.
sun-rui
## How was this patch tested?
1. SparkSubmitSuite
2. On local mesos cluster (on laptop): ran SparkR shell, spark-submit client mode, and spark-submit cluster mode, with the "examples/src/main/R/dataframe.R" example application.
3. On multi-node mesos cluster: ran SparkR shell, spark-submit client mode, and spark-submit cluster mode, with the "examples/src/main/R/dataframe.R" example application. I tested with the following --conf values set: spark.mesos.executor.docker.image and spark.mesos.executor.home
This contribution is my original work and I license the work to the project under the project's open source license.
Author: Susan X. Huynh <xhuynh@mesosphere.com>
Closes #15700 from susanxhuynh/susan-r-branch.
(cherry picked from commit 9a87c313859a6557bbf7bca7239043cb77ea23be)
Signed-off-by: Sean Owen <sowen@cloudera.com>
---
.../main/scala/org/apache/spark/api/r/RUtils.scala | 1 -
.../org/apache/spark/deploy/SparkSubmit.scala | 14 +++++++-------
2 files changed, 7 insertions(+), 8 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
index 77825e75e5..fdd8cf62f0 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RUtils.scala
@@ -84,7 +84,6 @@ private[spark] object RUtils {
}
} else {
// Otherwise, assume the package is local
- // TODO: support this for Mesos
val sparkRPkgPath = localSparkRPackagePath.getOrElse {
throw new SparkException("SPARK_HOME not set. Can't locate SparkR package.")
}
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index 5c05228609..c70061bc5b 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -322,7 +322,7 @@ object SparkSubmit {
}
// Require all R files to be local
- if (args.isR && !isYarnCluster) {
+ if (args.isR && !isYarnCluster && !isMesosCluster) {
if (Utils.nonLocalPaths(args.primaryResource).nonEmpty) {
printErrorAndExit(s"Only local R files are supported: ${args.primaryResource}")
}
@@ -330,9 +330,6 @@ object SparkSubmit {
// The following modes are not supported or applicable
(clusterManager, deployMode) match {
- case (MESOS, CLUSTER) if args.isR =>
- printErrorAndExit("Cluster deploy mode is currently not supported for R " +
- "applications on Mesos clusters.")
case (STANDALONE, CLUSTER) if args.isPython =>
printErrorAndExit("Cluster deploy mode is currently not supported for python " +
"applications on standalone clusters.")
@@ -410,9 +407,9 @@ object SparkSubmit {
printErrorAndExit("Distributing R packages with standalone cluster is not supported.")
}
- // TODO: Support SparkR with mesos cluster
- if (args.isR && clusterManager == MESOS) {
- printErrorAndExit("SparkR is not supported for Mesos cluster.")
+ // TODO: Support distributing R packages with mesos cluster
+ if (args.isR && clusterManager == MESOS && !RUtils.rPackages.isEmpty) {
+ printErrorAndExit("Distributing R packages with mesos cluster is not supported.")
}
// If we're running an R app, set the main class to our specific R runner
@@ -598,6 +595,9 @@ object SparkSubmit {
if (args.pyFiles != null) {
sysProps("spark.submit.pyFiles") = args.pyFiles
}
+ } else if (args.isR) {
+ // Second argument is main class
+ childArgs += (args.primaryResource, "")
} else {
childArgs += (args.primaryResource, args.mainClass)
}
--
GitLab