From db5037973728f53f234fe3381ba36122e1f47bba Mon Sep 17 00:00:00 2001
From: Patrick Wendell <pwendell@gmail.com>
Date: Fri, 7 Jun 2013 11:54:16 -0700
Subject: [PATCH] Decoupling spark version from ami
---
ec2/spark_ec2.py | 23 ++++++++++++++++-------
1 file changed, 16 insertions(+), 7 deletions(-)
diff --git a/ec2/spark_ec2.py b/ec2/spark_ec2.py
index 7df7ae2ae4..89bb12f157 100755
--- a/ec2/spark_ec2.py
+++ b/ec2/spark_ec2.py
@@ -36,7 +36,7 @@ from boto import ec2
# A URL prefix from which to fetch AMI information
AMI_PREFIX = "https://raw.github.com/pwendell/spark-ec2/ec2-updates/ami-list"
-
+LATEST_SPARK_VERSION = "0.7.2"
# Configure and parse our command-line arguments
def parse_args():
@@ -67,6 +67,8 @@ def parse_args():
parser.add_option("-a", "--ami", default="latest",
help="Amazon Machine Image ID to use, 'vX.Y.Z' to use version " +
"X.Y.Z of Spark, or 'latest' to use latest AMI (default: latest)")
+ parser.add_option("-v", "--spark-version", default="latest",
+ help="Version of Spark to use (X.Y.Z or 'latest' to use most recent)")
parser.add_option("-D", metavar="[ADDRESS:]PORT", dest="proxy_port",
help="Use SSH dynamic port forwarding to create a SOCKS proxy at " +
"the given local address (for use with login)")
@@ -156,6 +158,16 @@ def wait_for_instances(conn, instances):
def is_active(instance):
return (instance.state in ['pending', 'running', 'stopping', 'stopped'])
+# Return correct versions of Spark and Shark, given the supplied spark version
+def get_spark_shark_version(opts):
+ spark_shark_map = ["0.7.2", "0.7.0"]
+ version = opts.spark_version.replace("v", "")
+ if version not in ["latest", "0.7.2"]:
+ print >> stderr, "Don't know about spark version: %s" % version
+ if version == "latest":
+ version = LATEST_SPARK_VERSION
+ return (version, spark_shark_map[version])
+
# Attempt to resolve an appropriate AMI given the architecture and
# region of the request.
def get_spark_ami(opts):
@@ -187,11 +199,7 @@ def get_spark_ami(opts):
print >> stderr,\
"Don't recognize %s, assuming type is pvm" % opts.instance_type
- version = version.replace("v", "")
- if version not in ["latest", "0.7.0"]:
- print >> stderr, \
- "Don't know how to resolve AMI for version: %s" % version
- ami_path = "%s/%s/%s/%s" % (AMI_PREFIX, version, opts.region, instance_type)
+ ami_path = "%s/%s/%s" % (AMI_PREFIX, opts.region, instance_type)
try:
ami = urllib2.urlopen(ami_path).read().strip()
print "Spark AMI: " + ami
@@ -452,8 +460,9 @@ def setup_standalone_cluster(master, slave_nodes, opts):
ssh(master, opts, "/root/spark/bin/start-all.sh")
def setup_spark_cluster(master, opts):
+ (spark_v, shark_v) = get_spark_shark_version(opts)
ssh(master, opts, "chmod u+x spark-ec2/setup.sh")
- ssh(master, opts, "spark-ec2/setup.sh")
+ ssh(master, opts, "spark-ec2/setup.sh %s %s" % (spark_v, shark_v)))
if opts.cluster_type == "mesos":
print "Mesos cluster started at http://%s:8080" % master
elif opts.cluster_type == "standalone":
--
GitLab