From 0760787da885187b0c6dcd5c28753f0ab014d5ed Mon Sep 17 00:00:00 2001
From: WangTaoTheTonic <barneystinson@aliyun.com>
Date: Thu, 8 Jan 2015 11:45:42 -0800
Subject: [PATCH] [SPARK-5130][Deploy]Take yarn-cluster as cluster mode in
spark-submit
https://issues.apache.org/jira/browse/SPARK-5130
Author: WangTaoTheTonic <barneystinson@aliyun.com>
Closes #3929 from WangTaoTheTonic/SPARK-5130 and squashes the following commits:
c490648 [WangTaoTheTonic] take yarn-cluster as cluster mode in spark-submit
---
bin/spark-submit | 5 +++++
bin/spark-submit2.cmd | 6 ++++++
2 files changed, 11 insertions(+)
diff --git a/bin/spark-submit b/bin/spark-submit
index f92d90c3a6..aefd38a0a2 100755
--- a/bin/spark-submit
+++ b/bin/spark-submit
@@ -38,11 +38,16 @@ while (($#)); do
export SPARK_SUBMIT_CLASSPATH=$2
elif [ "$1" = "--driver-java-options" ]; then
export SPARK_SUBMIT_OPTS=$2
+ elif [ "$1" = "--master" ]; then
+ export MASTER=$2
fi
shift
done
DEFAULT_PROPERTIES_FILE="$SPARK_HOME/conf/spark-defaults.conf"
+if [ "$MASTER" == "yarn-cluster" ]; then
+ SPARK_SUBMIT_DEPLOY_MODE=cluster
+fi
export SPARK_SUBMIT_DEPLOY_MODE=${SPARK_SUBMIT_DEPLOY_MODE:-"client"}
export SPARK_SUBMIT_PROPERTIES_FILE=${SPARK_SUBMIT_PROPERTIES_FILE:-"$DEFAULT_PROPERTIES_FILE"}
diff --git a/bin/spark-submit2.cmd b/bin/spark-submit2.cmd
index cf6046d154..daf0284db9 100644
--- a/bin/spark-submit2.cmd
+++ b/bin/spark-submit2.cmd
@@ -45,11 +45,17 @@ if [%1] == [] goto continue
set SPARK_SUBMIT_CLASSPATH=%2
) else if [%1] == [--driver-java-options] (
set SPARK_SUBMIT_OPTS=%2
+ ) else if [%1] == [--master] (
+ set MASTER=%2
)
shift
goto loop
:continue
+if [%MASTER%] == [yarn-cluster] (
+ set SPARK_SUBMIT_DEPLOY_MODE=cluster
+)
+
rem For client mode, the driver will be launched in the same JVM that launches
rem SparkSubmit, so we may need to read the properties file for any extra class
rem paths, library paths, java options and memory early on. Otherwise, it will
--
GitLab