From 9f7a095184d6c7a9b1bbac55efcc3d878f876768 Mon Sep 17 00:00:00 2001
From: Patrick Wendell <pwendell@gmail.com>
Date: Mon, 28 Apr 2014 18:14:59 -0700
Subject: [PATCH] SPARK-1652: Remove incorrect deprecation warning in
spark-submit
This is a straightforward fix.
Author: Patrick Wendell <pwendell@gmail.com>
This patch had conflicts when merged, resolved by
Committer: Patrick Wendell <pwendell@gmail.com>
Closes #578 from pwendell/spark-submit-yarn and squashes the following commits:
96027c7 [Patrick Wendell] Test fixes
b5be173 [Patrick Wendell] Review feedback
4ac9cac [Patrick Wendell] SPARK-1652: spark-submit for yarn prints warnings even though calling as expected
---
.../main/scala/org/apache/spark/deploy/SparkSubmit.scala | 3 +++
.../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 5 +++--
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 6 ++++--
.../main/scala/org/apache/spark/deploy/yarn/Client.scala | 6 ++++--
4 files changed, 14 insertions(+), 6 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
index c463ee0999..d131f1809c 100644
--- a/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/SparkSubmit.scala
@@ -137,6 +137,9 @@ object SparkSubmit {
throw new Exception(msg)
}
}
+
+ // Special flag to avoid deprecation warnings at the client
+ sysProps("SPARK_SUBMIT") = "true"
val options = List[OptionAssigner](
new OptionAssigner(appArgs.master, ALL_CLUSTER_MGRS, false, sysProp = "spark.master"),
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 10a65c75cc..b3541b4a40 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -122,7 +122,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers {
childArgsStr should include ("--num-executors 6")
mainClass should be ("org.apache.spark.deploy.yarn.Client")
classpath should have length (0)
- sysProps should have size (0)
+ sysProps should have size (1)
}
test("handles YARN client mode") {
@@ -146,6 +146,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers {
sysProps("spark.yarn.dist.files") should be ("file1.txt,file2.txt")
sysProps("spark.yarn.dist.archives") should be ("archive1.txt,archive2.txt")
sysProps("spark.executor.instances") should be ("6")
+ sysProps("SPARK_SUBMIT") should be ("true")
}
test("handles standalone cluster mode") {
@@ -159,7 +160,7 @@ class SparkSubmitSuite extends FunSuite with ShouldMatchers {
childArgsStr should include ("launch spark://h:p thejar.jar org.SomeClass arg1 arg2")
mainClass should be ("org.apache.spark.deploy.Client")
classpath should have length (0)
- sysProps should have size (1) // contains --jar entry
+ sysProps should have size (2) // contains --jar entry and SPARK_SUBMIT
}
test("handles standalone client mode") {
diff --git a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 00c7649e68..8226207de4 100644
--- a/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/alpha/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -169,8 +169,10 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
object Client {
def main(argStrings: Array[String]) {
- println("WARNING: This client is deprecated and will be removed in a future version of Spark.")
- println("Use ./bin/spark-submit with \"--master yarn\"")
+ if (!sys.props.contains("SPARK_SUBMIT")) {
+ println("WARNING: This client is deprecated and will be removed in a " +
+ "future version of Spark. Use ./bin/spark-submit with \"--master yarn\"")
+ }
// Set an env variable indicating we are running in YARN mode.
// Note that anything with SPARK prefix gets propagated to all (remote) processes
diff --git a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
index 2e2fb5d4fa..1b6bfb42a5 100644
--- a/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
+++ b/yarn/stable/src/main/scala/org/apache/spark/deploy/yarn/Client.scala
@@ -171,8 +171,10 @@ class Client(clientArgs: ClientArguments, hadoopConf: Configuration, spConf: Spa
object Client {
def main(argStrings: Array[String]) {
- println("WARNING: This client is deprecated and will be removed in a future version of Spark.")
- println("Use ./bin/spark-submit with \"--master yarn\"")
+ if (!sys.props.contains("SPARK_SUBMIT")) {
+ println("WARNING: This client is deprecated and will be removed in a " +
+ "future version of Spark. Use ./bin/spark-submit with \"--master yarn\"")
+ }
// Set an env variable indicating we are running in YARN mode.
// Note: anything env variable with SPARK_ prefix gets propagated to all (remote) processes -
--
GitLab