From aec4400beffc569c13cceea2d0c481dfa3f34175 Mon Sep 17 00:00:00 2001
From: Jeff Zhang <zjffdu@apache.org>
Date: Thu, 15 Oct 2015 09:49:19 -0700
Subject: [PATCH] =?UTF-8?q?[SPARK-11099]=20[SPARK=20SHELL]=20[SPARK=20SUBM?=
=?UTF-8?q?IT]=20Default=20conf=20property=20file=20i=E2=80=A6?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Please help review it. Thanks
Author: Jeff Zhang <zjffdu@apache.org>
Closes #9114 from zjffdu/SPARK-11099.
---
.../launcher/AbstractCommandBuilder.java | 14 ++++------
.../SparkSubmitCommandBuilderSuite.java | 28 +++++++++++++------
.../src/test/resources/spark-defaults.conf | 21 ++++++++++++++
3 files changed, 45 insertions(+), 18 deletions(-)
create mode 100644 launcher/src/test/resources/spark-defaults.conf
diff --git a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
index cf3729b7fe..3ee6bd92e4 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/AbstractCommandBuilder.java
@@ -272,15 +272,11 @@ abstract class AbstractCommandBuilder {
Map<String, String> getEffectiveConfig() throws IOException {
if (effectiveConfig == null) {
- if (propertiesFile == null) {
- effectiveConfig = conf;
- } else {
- effectiveConfig = new HashMap<>(conf);
- Properties p = loadPropertiesFile();
- for (String key : p.stringPropertyNames()) {
- if (!effectiveConfig.containsKey(key)) {
- effectiveConfig.put(key, p.getProperty(key));
- }
+ effectiveConfig = new HashMap<>(conf);
+ Properties p = loadPropertiesFile();
+ for (String key : p.stringPropertyNames()) {
+ if (!effectiveConfig.containsKey(key)) {
+ effectiveConfig.put(key, p.getProperty(key));
}
}
}
diff --git a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
index d5397b0685..6aad47adbc 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/SparkSubmitCommandBuilderSuite.java
@@ -48,12 +48,14 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
@Test
public void testDriverCmdBuilder() throws Exception {
- testCmdBuilder(true);
+ testCmdBuilder(true, true);
+ testCmdBuilder(true, false);
}
@Test
public void testClusterCmdBuilder() throws Exception {
- testCmdBuilder(false);
+ testCmdBuilder(false, true);
+ testCmdBuilder(false, false);
}
@Test
@@ -149,7 +151,7 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
assertEquals("arg1", cmd.get(cmd.size() - 1));
}
- private void testCmdBuilder(boolean isDriver) throws Exception {
+ private void testCmdBuilder(boolean isDriver, boolean useDefaultPropertyFile) throws Exception {
String deployMode = isDriver ? "client" : "cluster";
SparkSubmitCommandBuilder launcher =
@@ -161,14 +163,20 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
launcher.appResource = "/foo";
launcher.appName = "MyApp";
launcher.mainClass = "my.Class";
- launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath());
launcher.appArgs.add("foo");
launcher.appArgs.add("bar");
- launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g");
- launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver");
- launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m");
- launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native");
launcher.conf.put("spark.foo", "foo");
+ // either set the property through "--conf" or through default property file
+ if (!useDefaultPropertyFile) {
+ launcher.setPropertiesFile(dummyPropsFile.getAbsolutePath());
+ launcher.conf.put(SparkLauncher.DRIVER_MEMORY, "1g");
+ launcher.conf.put(SparkLauncher.DRIVER_EXTRA_CLASSPATH, "/driver");
+ launcher.conf.put(SparkLauncher.DRIVER_EXTRA_JAVA_OPTIONS, "-Ddriver -XX:MaxPermSize=256m");
+ launcher.conf.put(SparkLauncher.DRIVER_EXTRA_LIBRARY_PATH, "/native");
+ } else {
+ launcher.childEnv.put("SPARK_CONF_DIR", System.getProperty("spark.test.home")
+ + "/launcher/src/test/resources");
+ }
Map<String, String> env = new HashMap<String, String>();
List<String> cmd = launcher.buildCommand(env);
@@ -216,7 +224,9 @@ public class SparkSubmitCommandBuilderSuite extends BaseSuite {
}
// Checks below are the same for both driver and non-driver mode.
- assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE));
+ if (!useDefaultPropertyFile) {
+ assertEquals(dummyPropsFile.getAbsolutePath(), findArgValue(cmd, parser.PROPERTIES_FILE));
+ }
assertEquals("yarn", findArgValue(cmd, parser.MASTER));
assertEquals(deployMode, findArgValue(cmd, parser.DEPLOY_MODE));
assertEquals("my.Class", findArgValue(cmd, parser.CLASS));
diff --git a/launcher/src/test/resources/spark-defaults.conf b/launcher/src/test/resources/spark-defaults.conf
new file mode 100644
index 0000000000..239fc57883
--- /dev/null
+++ b/launcher/src/test/resources/spark-defaults.conf
@@ -0,0 +1,21 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+spark.driver.memory=1g
+spark.driver.extraClassPath=/driver
+spark.driver.extraJavaOptions=-Ddriver -XX:MaxPermSize=256m
+spark.driver.extraLibraryPath=/native
\ No newline at end of file
--
GitLab