From ae4ea5fe253e9083e807bdc769e829f3f37265b4 Mon Sep 17 00:00:00 2001
From: Yuhao Yang <yuhao.yang@intel.com>
Date: Wed, 26 Jul 2017 10:37:48 +0100
Subject: [PATCH] [SPARK-21524][ML] unit test fix: ValidatorParamsSuiteHelpers
generates wrong temp files
## What changes were proposed in this pull request?
jira: https://issues.apache.org/jira/browse/SPARK-21524
ValidatorParamsSuiteHelpers.testFileMove() is generating temp dir in the wrong place and does not delete them.
ValidatorParamsSuiteHelpers.testFileMove() is invoked by TrainValidationSplitSuite and crossValidatorSuite. Currently it uses `tempDir` from `TempDirectory`, which unfortunately is never initialized since the `boforeAll()` of `ValidatorParamsSuiteHelpers` is never invoked.
In my system, it leaves some temp directories in the assembly folder each time I run the TrainValidationSplitSuite and crossValidatorSuite.
## How was this patch tested?
unit test fix
Author: Yuhao Yang <yuhao.yang@intel.com>
Closes #18728 from hhbyyh/tempDirFix.
---
.../org/apache/spark/ml/tuning/CrossValidatorSuite.scala | 2 +-
.../spark/ml/tuning/TrainValidationSplitSuite.scala | 2 +-
.../spark/ml/tuning/ValidatorParamsSuiteHelpers.scala | 9 +++++----
3 files changed, 7 insertions(+), 6 deletions(-)
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
index 2791ea715a..dc6043ef19 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
@@ -222,7 +222,7 @@ class CrossValidatorSuite
.setNumFolds(20)
.setEstimatorParamMaps(paramMaps)
- ValidatorParamsSuiteHelpers.testFileMove(cv)
+ ValidatorParamsSuiteHelpers.testFileMove(cv, tempDir)
}
test("read/write: CrossValidator with complex estimator") {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala
index 71a1776a2c..7c97865e45 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala
@@ -209,7 +209,7 @@ class TrainValidationSplitSuite
.setEstimatorParamMaps(paramMaps)
.setSeed(42L)
- ValidatorParamsSuiteHelpers.testFileMove(tvs)
+ ValidatorParamsSuiteHelpers.testFileMove(tvs, tempDir)
}
test("read/write: TrainValidationSplitModel") {
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala
index 1df673cf40..eae1f5adc8 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala
@@ -20,11 +20,12 @@ package org.apache.spark.ml.tuning
import java.io.File
import java.nio.file.{Files, StandardCopyOption}
-import org.apache.spark.SparkFunSuite
+import org.scalatest.Assertions
+
import org.apache.spark.ml.param.{ParamMap, ParamPair, Params}
-import org.apache.spark.ml.util.{DefaultReadWriteTest, Identifiable, MLReader, MLWritable}
+import org.apache.spark.ml.util.{Identifiable, MLReader, MLWritable}
-object ValidatorParamsSuiteHelpers extends SparkFunSuite with DefaultReadWriteTest {
+object ValidatorParamsSuiteHelpers extends Assertions {
/**
* Assert sequences of estimatorParamMaps are identical.
* If the values for a parameter are not directly comparable with ===
@@ -62,7 +63,7 @@ object ValidatorParamsSuiteHelpers extends SparkFunSuite with DefaultReadWriteTe
* the path of the estimator so that if the parent directory changes, loading the
* model still works.
*/
- def testFileMove[T <: Params with MLWritable](instance: T): Unit = {
+ def testFileMove[T <: Params with MLWritable](instance: T, tempDir: File): Unit = {
val uid = instance.uid
val subdirName = Identifiable.randomUID("test")
--
GitLab