From e09e18b3123c20e9b9497cf606473da500349d4d Mon Sep 17 00:00:00 2001
From: Andrew Or <andrewor14@gmail.com>
Date: Sat, 2 Aug 2014 12:11:50 -0700
Subject: [PATCH] [HOTFIX] Do not throw NPE if spark.test.home is not set
`spark.test.home` was introduced in #1734. This is fine for SBT but is failing maven tests. Either way it shouldn't throw an NPE.
Author: Andrew Or <andrewor14@gmail.com>
Closes #1739 from andrewor14/fix-spark-test-home and squashes the following commits:
ce2624c [Andrew Or] Do not throw NPE if spark.test.home is not set
---
.../scala/org/apache/spark/deploy/worker/Worker.scala | 9 +++++++--
core/src/test/scala/org/apache/spark/DriverSuite.scala | 2 +-
.../scala/org/apache/spark/deploy/SparkSubmitSuite.scala | 2 +-
.../apache/spark/deploy/worker/ExecutorRunnerTest.scala | 2 +-
pom.xml | 8 ++++----
5 files changed, 14 insertions(+), 9 deletions(-)
diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
index c6ea42fceb..458d9947bd 100755
--- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
+++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala
@@ -71,7 +71,7 @@ private[spark] class Worker(
// TTL for app folders/data; after TTL expires it will be cleaned up
val APP_DATA_RETENTION_SECS = conf.getLong("spark.worker.cleanup.appDataTtl", 7 * 24 * 3600)
-
+ val testing: Boolean = sys.props.contains("spark.testing")
val masterLock: Object = new Object()
var master: ActorSelection = null
var masterAddress: Address = null
@@ -82,7 +82,12 @@ private[spark] class Worker(
@volatile var connected = false
val workerId = generateWorkerId()
val sparkHome =
- new File(sys.props.get("spark.test.home").orElse(sys.env.get("SPARK_HOME")).getOrElse("."))
+ if (testing) {
+ assert(sys.props.contains("spark.test.home"), "spark.test.home is not set!")
+ new File(sys.props("spark.test.home"))
+ } else {
+ new File(sys.env.get("SPARK_HOME").getOrElse("."))
+ }
var workDir: File = null
val executors = new HashMap[String, ExecutorRunner]
val finishedExecutors = new HashMap[String, ExecutorRunner]
diff --git a/core/src/test/scala/org/apache/spark/DriverSuite.scala b/core/src/test/scala/org/apache/spark/DriverSuite.scala
index e36902ec81..a73e1ef028 100644
--- a/core/src/test/scala/org/apache/spark/DriverSuite.scala
+++ b/core/src/test/scala/org/apache/spark/DriverSuite.scala
@@ -34,7 +34,7 @@ import scala.language.postfixOps
class DriverSuite extends FunSuite with Timeouts {
test("driver should exit after finishing") {
- val sparkHome = sys.props("spark.test.home")
+ val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
// Regression test for SPARK-530: "Spark driver process doesn't exit after finishing"
val masters = Table(("master"), ("local"), ("local-cluster[2,1,512]"))
forAll(masters) { (master: String) =>
diff --git a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
index 8126ef1bb2..a5cdcfb5de 100644
--- a/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/SparkSubmitSuite.scala
@@ -295,7 +295,7 @@ class SparkSubmitSuite extends FunSuite with Matchers {
// NOTE: This is an expensive operation in terms of time (10 seconds+). Use sparingly.
def runSparkSubmit(args: Seq[String]): String = {
- val sparkHome = sys.props("spark.test.home")
+ val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
Utils.executeAndGetOutput(
Seq("./bin/spark-submit") ++ args,
new File(sparkHome),
diff --git a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
index 149a2b3d95..39ab53cf0b 100644
--- a/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
+++ b/core/src/test/scala/org/apache/spark/deploy/worker/ExecutorRunnerTest.scala
@@ -27,7 +27,7 @@ import org.apache.spark.SparkConf
class ExecutorRunnerTest extends FunSuite {
test("command includes appId") {
def f(s:String) = new File(s)
- val sparkHome = sys.props("spark.test.home")
+ val sparkHome = sys.props.getOrElse("spark.test.home", fail("spark.test.home is not set!"))
val appDesc = new ApplicationDescription("app name", Some(8), 500,
Command("foo", Seq(), Map(), Seq(), Seq(), Seq()), "appUiUrl")
val appId = "12345-worker321-9876"
diff --git a/pom.xml b/pom.xml
index ae97bf03c5..99ae4b8b33 100644
--- a/pom.xml
+++ b/pom.xml
@@ -868,10 +868,10 @@
<filereports>${project.build.directory}/SparkTestSuite.txt</filereports>
<argLine>-Xmx3g -XX:MaxPermSize=${MaxPermGen} -XX:ReservedCodeCacheSize=512m</argLine>
<stderr/>
- <environmentVariables>
- <SPARK_HOME>${session.executionRootDirectory}</SPARK_HOME>
- <SPARK_TESTING>1</SPARK_TESTING>
- </environmentVariables>
+ <systemProperties>
+ <spark.test.home>${session.executionRootDirectory}</spark.test.home>
+ <spark.testing>1</spark.testing>
+ </systemProperties>
</configuration>
<executions>
<execution>
--
GitLab