From 01cf4c402b9fda59680e56112bfaa2b748416d0e Mon Sep 17 00:00:00 2001
From: Aaron Davidson <aaron@databricks.com>
Date: Fri, 4 Apr 2014 09:50:24 -0700
Subject: [PATCH] SPARK-1404: Always upgrade spark-env.sh vars to environment
vars
This was broken when spark-env.sh was made idempotent, as the idempotence check is an environment variable, but the spark-env.sh variables may not have been.
Tested in zsh, bash, and sh.
Author: Aaron Davidson <aaron@databricks.com>
Closes #310 from aarondav/SPARK-1404 and squashes the following commits:
c3406a5 [Aaron Davidson] Add extra export in spark-shell
6a0e340 [Aaron Davidson] SPARK-1404: Always upgrade spark-env.sh vars to environment vars
---
bin/load-spark-env.sh | 3 +++
bin/spark-shell | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/bin/load-spark-env.sh b/bin/load-spark-env.sh
index 476dd82655..d425f9feaa 100644
--- a/bin/load-spark-env.sh
+++ b/bin/load-spark-env.sh
@@ -30,6 +30,9 @@ if [ -z "$SPARK_ENV_LOADED" ]; then
use_conf_dir=${SPARK_CONF_DIR:-"$parent_dir/conf"}
if [ -f "${use_conf_dir}/spark-env.sh" ]; then
+ # Promote all variable declarations to environment (exported) variables
+ set -a
. "${use_conf_dir}/spark-env.sh"
+ set +a
fi
fi
diff --git a/bin/spark-shell b/bin/spark-shell
index fac006cf49..535ee3ccd8 100755
--- a/bin/spark-shell
+++ b/bin/spark-shell
@@ -127,7 +127,7 @@ function set_spark_log_conf(){
function set_spark_master(){
if ! [[ "$1" =~ $ARG_FLAG_PATTERN ]]; then
- MASTER="$1"
+ export MASTER="$1"
else
out_error "wrong format for $2"
fi
@@ -145,7 +145,7 @@ function resolve_spark_master(){
fi
if [ -z "$MASTER" ]; then
- MASTER="$DEFAULT_MASTER"
+ export MASTER="$DEFAULT_MASTER"
fi
}
--
GitLab