From 7c7d2d5e093b0623edc75dd166ec1179b4e62062 Mon Sep 17 00:00:00 2001
From: cheng chang <myairia@gmail.com>
Date: Tue, 10 Mar 2015 11:02:12 +0000
Subject: [PATCH] [SPARK-6186] [EC2] Make Tachyon version configurable in EC2
deployment script
This PR comes from Tachyon community to solve the issue:
https://tachyon.atlassian.net/browse/TACHYON-11
An accompanying PR is in mesos/spark-ec2:
https://github.com/mesos/spark-ec2/pull/101
Author: cheng chang <myairia@gmail.com>
Closes #4901 from uronce-cc/master and squashes the following commits:
313aa36 [cheng chang] minor re-wording
fd2a48e [cheng chang] Remove Tachyon when deploying through git hash
1d53c5c [cheng chang] add default value to --tachyon-version
6f8887e [cheng chang] make tachyon version configurable
---
.../root/spark-ec2/ec2-variables.sh | 3 ++-
ec2/spark_ec2.py | 19 +++++++++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh
index 740c267fd9..0857657152 100644
--- a/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh
+++ b/ec2/deploy.generic/root/spark-ec2/ec2-variables.sh
@@ -26,9 +26,10 @@ export SPARK_LOCAL_DIRS="{{spark_local_dirs}}"
export MODULES="{{modules}}"
export SPARK_VERSION="{{spark_version}}"
export SHARK_VERSION="{{shark_version}}"
+export TACHYON_VERSION="{{tachyon_version}}"
export HADOOP_MAJOR_VERSION="{{hadoop_major_version}}"
export SWAP_MB="{{swap}}"
export SPARK_WORKER_INSTANCES="{{spark_worker_instances}}"
export SPARK_MASTER_OPTS="{{spark_master_opts}}"
export AWS_ACCESS_KEY_ID="{{aws_access_key_id}}"
-export AWS_SECRET_ACCESS_KEY="{{aws_secret_access_key}}"
\ No newline at end of file
+export AWS_SECRET_ACCESS_KEY="{{aws_secret_access_key}}"
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py
index 3acb5fea04..f848874b0c 100755
--- a/ec2/spark_ec2.py
+++ b/ec2/spark_ec2.py
@@ -62,6 +62,16 @@ VALID_SPARK_VERSIONS = set([
"1.2.1",
])
+SPARK_TACHYON_MAP = {
+ "1.0.0": "0.4.1",
+ "1.0.1": "0.4.1",
+ "1.0.2": "0.4.1",
+ "1.1.0": "0.5.0",
+ "1.1.1": "0.5.0",
+ "1.2.0": "0.5.0",
+ "1.2.1": "0.5.0",
+}
+
DEFAULT_SPARK_VERSION = SPARK_EC2_VERSION
DEFAULT_SPARK_GITHUB_REPO = "https://github.com/apache/spark"
@@ -370,6 +380,10 @@ EC2_INSTANCE_TYPES = {
}
+def get_tachyon_version(spark_version):
+ return SPARK_TACHYON_MAP.get(spark_version, "")
+
+
# Attempt to resolve an appropriate AMI given the architecture and region of the request.
def get_spark_ami(opts):
if opts.instance_type in EC2_INSTANCE_TYPES:
@@ -919,9 +933,13 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules):
if "." in opts.spark_version:
# Pre-built Spark deploy
spark_v = get_validate_spark_version(opts.spark_version, opts.spark_git_repo)
+ tachyon_v = get_tachyon_version(spark_v)
else:
# Spark-only custom deploy
spark_v = "%s|%s" % (opts.spark_git_repo, opts.spark_version)
+ tachyon_v = ""
+ print "Deploying Spark via git hash; Tachyon won't be set up"
+ modules = filter(lambda x: x != "tachyon", modules)
template_vars = {
"master_list": '\n'.join([i.public_dns_name for i in master_nodes]),
@@ -934,6 +952,7 @@ def deploy_files(conn, root_dir, opts, master_nodes, slave_nodes, modules):
"swap": str(opts.swap),
"modules": '\n'.join(modules),
"spark_version": spark_v,
+ "tachyon_version": tachyon_v,
"hadoop_major_version": opts.hadoop_major_version,
"spark_worker_instances": "%d" % opts.worker_instances,
"spark_master_opts": opts.master_opts
--
GitLab