From e9d268f63e7308486739aa56ece02815bfb432d6 Mon Sep 17 00:00:00 2001
From: Kent Yao <yaooqinn@hotmail.com>
Date: Thu, 30 Mar 2017 16:11:03 +0100
Subject: [PATCH] [SPARK-20096][SPARK SUBMIT][MINOR] Expose the right queue
name not null if set by --conf or configure file
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
## What changes were proposed in this pull request?
while submit apps with -v or --verbose， we can print the right queue name, but if we set a queue name with `spark.yarn.queue` by --conf or in the spark-default.conf, we just got `null` for the queue in Parsed arguments.
```
bin/spark-shell -v --conf spark.yarn.queue=thequeue
Using properties file: /home/hadoop/spark-2.1.0-bin-apache-hdp2.7.3/conf/spark-defaults.conf
....
Adding default property: spark.yarn.queue=default
Parsed arguments:
master yarn
deployMode client
...
queue null
....
verbose true
Spark properties used, including those specified through
--conf and those from the properties file /home/hadoop/spark-2.1.0-bin-apache-hdp2.7.3/conf/spark-defaults.conf:
spark.yarn.queue -> thequeue
....
```
## How was this patch tested?
ut and local verify
Author: Kent Yao <yaooqinn@hotmail.com>
Closes #17430 from yaooqinn/SPARK-20096.
---
.../apache/spark/deploy/SparkSubmitArguments.scala | 1 +
.../org/apache/spark/deploy/SparkSubmitSuite.scala | 11 +++++++++++
2 files changed, 12 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
index 0614d80b60..0144fd1056 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmitArguments.scala
@@ -190,6 +190,7 @@ private[deploy] class SparkSubmitArguments(args: Seq[String], env: Map[String, S
.orNull
numExecutors = Option(numExecutors)
.getOrElse(sparkProperties.get("spark.executor.instances").orNull)
+ queue = Option(queue).orElse(sparkProperties.get("spark.yarn.queue")).orNull
keytab = Option(keytab).orElse(sparkProperties.get("spark.yarn.keytab")).orNull
principal = Option(principal).orElse(sparkProperties.get("spark.yarn.principal")).orNull
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index a591b98bca..7c2ec01a03 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -148,6 +148,17 @@ class SparkSubmitSuite
appArgs.childArgs should be (Seq("--master", "local", "some", "--weird", "args"))
}
+ test("print the right queue name") {
+ val clArgs = Seq(
+ "--name", "myApp",
+ "--class", "Foo",
+ "--conf", "spark.yarn.queue=thequeue",
+ "userjar.jar")
+ val appArgs = new SparkSubmitArguments(clArgs)
+ appArgs.queue should be ("thequeue")
+ appArgs.toString should include ("thequeue")
+ }
+
test("specify deploy mode through configuration") {
val clArgs = Seq(
"--master", "yarn",
--
GitLab