From 53008c2d8a8b28f4204eaafa89f0e8087dc11466 Mon Sep 17 00:00:00 2001
From: Denny <dennybritz@gmail.com>
Date: Thu, 2 Aug 2012 15:59:39 -0700
Subject: [PATCH] Settings variables and bugfix for stop script.
---
bin/stop-master.sh | 2 +-
bin/stop-slaves.sh | 2 +-
conf/spark-env.sh.template | 10 +++++++++-
.../spark/deploy/master/MasterArguments.scala | 13 ++++++++++++-
.../spark/deploy/worker/WorkerArguments.scala | 16 +++++++++++++++-
5 files changed, 38 insertions(+), 5 deletions(-)
diff --git a/bin/stop-master.sh b/bin/stop-master.sh
index f33f71664e..f75167dd2c 100755
--- a/bin/stop-master.sh
+++ b/bin/stop-master.sh
@@ -7,4 +7,4 @@ bin=`cd "$bin"; pwd`
. "$bin/spark-config.sh"
-"$bin"/spark-daemon.sh stop spark.deploy.worker.Worker
\ No newline at end of file
+"$bin"/spark-daemon.sh stop spark.deploy.master.Master
\ No newline at end of file
diff --git a/bin/stop-slaves.sh b/bin/stop-slaves.sh
index f75167dd2c..21c9ebf324 100755
--- a/bin/stop-slaves.sh
+++ b/bin/stop-slaves.sh
@@ -7,4 +7,4 @@ bin=`cd "$bin"; pwd`
. "$bin/spark-config.sh"
-"$bin"/spark-daemon.sh stop spark.deploy.master.Master
\ No newline at end of file
+"$bin"/spark-daemons.sh stop spark.deploy.worker.Worker
\ No newline at end of file
diff --git a/conf/spark-env.sh.template b/conf/spark-env.sh.template
index c09af42717..64eacce8a2 100755
--- a/conf/spark-env.sh.template
+++ b/conf/spark-env.sh.template
@@ -9,5 +9,13 @@
# - SPARK_MEM, to change the amount of memory used per node (this should
# be in the same format as the JVM's -Xmx option, e.g. 300m or 1g).
# - SPARK_LIBRARY_PATH, to add extra search paths for native libraries.
-# - SPARK_MASTER_PORT, to start the spark master on a different port (standalone mode only)
+# Settings used by the scripts in the bin/ directory, apply to standalone mode only.
+# Note that the same worker settings apply to all of the workers.
+# - SPARK_MASTER_IP, to bind the master to a different ip address, for example a public one (Default: local ip address)
+# - SPARK_MASTER_PORT, to start the spark master on a different port (Default: 7077)
+# - SPARK_MASTER_WEBUI_PORT, to specify a different port for the Master WebUI (Default: 8080)
+# - SPARK_WORKER_PORT, to start the spark worker on a specific port (Default: random)
+# - SPARK_WORKER_CORES, to specify the number of cores to use (Default: all available cores)
+# - SPARK_WORKER_MEMORY, to specify how much memory to use, e.g. 1000M, 2G (Default: MAX(Available - 1024MB, 512MB))
+# - SPARK_WORKER_WEBUI_PORT, to specify a different port for the Worker WebUI (Default: 8081)
\ No newline at end of file
diff --git a/core/src/main/scala/spark/deploy/master/MasterArguments.scala b/core/src/main/scala/spark/deploy/master/MasterArguments.scala
index 5d975cd546..0f7a92bdd0 100644
--- a/core/src/main/scala/spark/deploy/master/MasterArguments.scala
+++ b/core/src/main/scala/spark/deploy/master/MasterArguments.scala
@@ -10,7 +10,18 @@ class MasterArguments(args: Array[String]) {
var ip = Utils.localIpAddress()
var port = 7077
var webUiPort = 8080
-
+
+ // Check for settings in environment variables
+ if (System.getenv("SPARK_MASTER_IP") != null) {
+ ip = System.getenv("SPARK_MASTER_IP")
+ }
+ if (System.getenv("SPARK_MASTER_PORT") != null) {
+ port = System.getenv("SPARK_MASTER_PORT").toInt
+ }
+ if (System.getenv("SPARK_MASTER_WEBUI_PORT") != null) {
+ webUiPort = System.getenv("SPARK_MASTER_WEBUI_PORT").toInt
+ }
+
parse(args.toList)
def parse(args: List[String]): Unit = args match {
diff --git a/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala b/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala
index 3248d03697..1efe8304ea 100644
--- a/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala
+++ b/core/src/main/scala/spark/deploy/worker/WorkerArguments.scala
@@ -15,7 +15,21 @@ class WorkerArguments(args: Array[String]) {
var cores = inferDefaultCores()
var memory = inferDefaultMemory()
var master: String = null
-
+
+ // Check for settings in environment variables
+ if (System.getenv("SPARK_WORKER_PORT") != null) {
+ port = System.getenv("SPARK_WORKER_PORT").toInt
+ }
+ if (System.getenv("SPARK_WORKER_CORES") != null) {
+ cores = System.getenv("SPARK_WORKER_CORES").toInt
+ }
+ if (System.getenv("SPARK_WORKER_MEMORY") != null) {
+ memory = Utils.memoryStringToMb(System.getenv("SPARK_WORKER_MEMORY"))
+ }
+ if (System.getenv("SPARK_WORKER_WEBUI_PORT") != null) {
+ webUiPort = System.getenv("SPARK_WORKER_WEBUI_PORT").toInt
+ }
+
parse(args.toList)
def parse(args: List[String]): Unit = args match {
--
GitLab