From 2d2411faa2dd1b7312c4277b2dd9e5678195cfbb Mon Sep 17 00:00:00 2001
From: Xiangrui Meng <meng@databricks.com>
Date: Fri, 13 Nov 2015 13:09:28 -0800
Subject: [PATCH] [SPARK-11672][ML] Set active SQLContext in
MLlibTestSparkContext.beforeAll
Still saw some error messages caused by `SQLContext.getOrCreate`:
https://amplab.cs.berkeley.edu/jenkins/job/Spark-Master-SBT/3997/AMPLAB_JENKINS_BUILD_PROFILE=hadoop2.3,label=spark-test/testReport/junit/org.apache.spark.ml.util/JavaDefaultReadWriteSuite/testDefaultReadWrite/
This PR sets the active SQLContext in beforeAll, which is not automatically set in `new SQLContext`. This makes `SQLContext.getOrCreate` return the right SQLContext.
cc: yhuai
Author: Xiangrui Meng <meng@databricks.com>
Closes #9694 from mengxr/SPARK-11672.3.
---
.../main/scala/org/apache/spark/ml/util/ReadWrite.scala | 7 +++++--
.../apache/spark/mllib/util/MLlibTestSparkContext.scala | 1 +
2 files changed, 6 insertions(+), 2 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
index 85f888c9f2..ca896ed610 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/util/ReadWrite.scala
@@ -48,8 +48,11 @@ private[util] sealed trait BaseReadWrite {
/**
* Returns the user-specified SQL context or the default.
*/
- protected final def sqlContext: SQLContext = optionSQLContext.getOrElse {
- SQLContext.getOrCreate(SparkContext.getOrCreate())
+ protected final def sqlContext: SQLContext = {
+ if (optionSQLContext.isEmpty) {
+ optionSQLContext = Some(SQLContext.getOrCreate(SparkContext.getOrCreate()))
+ }
+ optionSQLContext.get
}
/** Returns the [[SparkContext]] underlying [[sqlContext]] */
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala b/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala
index 998ee48186..378139593b 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/util/MLlibTestSparkContext.scala
@@ -34,6 +34,7 @@ trait MLlibTestSparkContext extends BeforeAndAfterAll { self: Suite =>
sc = new SparkContext(conf)
SQLContext.clearActive()
sqlContext = new SQLContext(sc)
+ SQLContext.setActive(sqlContext)
}
override def afterAll() {
--
GitLab