Skip to content
Snippets Groups Projects
Commit 8dbe7777 authored by meawoppl's avatar meawoppl Committed by Davies Liu
Browse files

[SPARK-7806][EC2] Fixes that allow the spark_ec2.py tool to run with Python3

I have used this script to launch, destroy, start, and stop clusters successfully.

Author: meawoppl <meawoppl@gmail.com>

Closes #6336 from meawoppl/py3ec2spark and squashes the following commits:

2e87046 [meawoppl] Py3 compat fixes.
parent 8948ad3f
No related branches found
No related tags found
No related merge requests found
......@@ -19,8 +19,9 @@
# limitations under the License.
#
from __future__ import with_statement, print_function
from __future__ import division, print_function, with_statement
import codecs
import hashlib
import itertools
import logging
......@@ -47,6 +48,8 @@ if sys.version < "3":
else:
from urllib.request import urlopen, Request
from urllib.error import HTTPError
raw_input = input
xrange = range
SPARK_EC2_VERSION = "1.3.1"
SPARK_EC2_DIR = os.path.dirname(os.path.realpath(__file__))
......@@ -423,13 +426,14 @@ def get_spark_ami(opts):
b=opts.spark_ec2_git_branch)
ami_path = "%s/%s/%s" % (ami_prefix, opts.region, instance_type)
reader = codecs.getreader("ascii")
try:
ami = urlopen(ami_path).read().strip()
print("Spark AMI: " + ami)
ami = reader(urlopen(ami_path)).read().strip()
except:
print("Could not resolve AMI at: " + ami_path, file=stderr)
sys.exit(1)
print("Spark AMI: " + ami)
return ami
......@@ -750,7 +754,7 @@ def setup_cluster(conn, master_nodes, slave_nodes, opts, deploy_ssh_key):
'mapreduce', 'spark-standalone', 'tachyon']
if opts.hadoop_major_version == "1":
modules = filter(lambda x: x != "mapreduce", modules)
modules = list(filter(lambda x: x != "mapreduce", modules))
if opts.ganglia:
modules.append('ganglia')
......@@ -1160,7 +1164,7 @@ def get_zones(conn, opts):
# Gets the number of items in a partition
def get_partition(total, num_partitions, current_partitions):
num_slaves_this_zone = total / num_partitions
num_slaves_this_zone = total // num_partitions
if (total % num_partitions) - current_partitions > 0:
num_slaves_this_zone += 1
return num_slaves_this_zone
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment