From bf8d0aa27844b1e58f131d49a3f668d1614ca6e7 Mon Sep 17 00:00:00 2001
From: Sandy Ryza <sandy@cloudera.com>
Date: Tue, 29 Apr 2014 12:54:02 -0700
Subject: [PATCH] SPARK-1588. Restore SPARK_YARN_USER_ENV and SPARK_JAVA_OPTS
for YARN.
Author: Sandy Ryza <sandy@cloudera.com>
Closes #586 from sryza/sandy-spark-1588 and squashes the following commits:
35eb38e [Sandy Ryza] Scalify
b361684 [Sandy Ryza] SPARK-1588. Restore SPARK_YARN_USER_ENV and SPARK_JAVA_OPTS for YARN.
---
.../apache/spark/deploy/yarn/ClientBase.scala | 16 +++++++++++++---
.../spark/deploy/yarn/ExecutorRunnableUtil.scala | 4 ++--
2 files changed, 15 insertions(+), 5 deletions(-)
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
index f2be8217a2..27a518ccda 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ClientBase.scala
@@ -263,9 +263,13 @@ trait ClientBase extends Logging {
distCacheMgr.setDistFilesEnv(env)
distCacheMgr.setDistArchivesEnv(env)
- // Allow users to specify some environment variables.
- YarnSparkHadoopUtil.setEnvFromInputString(env, System.getenv("SPARK_YARN_USER_ENV"),
- File.pathSeparator)
+ sys.env.get("SPARK_YARN_USER_ENV").foreach { userEnvs =>
+ // Allow users to specify some environment variables.
+ YarnSparkHadoopUtil.setEnvFromInputString(env, userEnvs, File.pathSeparator)
+
+ // Pass SPARK_YARN_USER_ENV itself to the AM so it can use it to set up executor environments.
+ env("SPARK_YARN_USER_ENV") = userEnvs
+ }
env
}
@@ -322,6 +326,12 @@ trait ClientBase extends Logging {
JAVA_OPTS += "-XX:CMSIncrementalDutyCycle=10"
}
+ // SPARK_JAVA_OPTS is deprecated, but for backwards compatibility:
+ sys.env.get("SPARK_JAVA_OPTS").foreach { opts =>
+ sparkConf.set("spark.executor.extraJavaOptions", opts)
+ sparkConf.set("spark.driver.extraJavaOptions", opts)
+ }
+
// TODO: it might be nicer to pass these as an internal environment variable rather than
// as Java options, due to complications with string parsing of nested quotes.
if (args.amClass == classOf[ExecutorLauncher].getName) {
diff --git a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
index 7d07f6f680..96f8aa9339 100644
--- a/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
+++ b/yarn/common/src/main/scala/org/apache/spark/deploy/yarn/ExecutorRunnableUtil.scala
@@ -71,8 +71,8 @@ trait ExecutorRunnableUtil extends Logging {
/*
else {
// If no java_opts specified, default to using -XX:+CMSIncrementalMode
- // It might be possible that other modes/config is being done in SPARK_JAVA_OPTS, so we dont
- // want to mess with it.
+ // It might be possible that other modes/config is being done in spark.executor.extraJavaOptions,
+ // so we dont want to mess with it.
// In our expts, using (default) throughput collector has severe perf ramnifications in
// multi-tennent machines
// The options are based on
--
GitLab