From ecf0c02935815f0d4018c0e30ec4c784e60a5db0 Mon Sep 17 00:00:00 2001
From: Prashant Sharma <prashant@apache.org>
Date: Mon, 15 Sep 2014 21:14:00 -0700
Subject: [PATCH] [SPARK-3433][BUILD] Fix for Mima false-positives with
@DeveloperAPI and @Experimental annotations.
Actually false positive reported was due to mima generator not picking up the new jars in presence of old jars(theoretically this should not have happened.). So as a workaround, ran them both separately and just append them together.
Author: Prashant Sharma <prashant@apache.org>
Author: Prashant Sharma <prashant.s@imaginea.com>
Closes #2285 from ScrapCodes/mima-fix and squashes the following commits:
093c76f [Prashant Sharma] Update mima
59012a8 [Prashant Sharma] Update mima
35b6c71 [Prashant Sharma] SPARK-3433 Fix for Mima false-positives with @DeveloperAPI and @Experimental annotations.
---
dev/mima | 8 ++++++++
project/MimaBuild.scala | 6 ++++++
project/MimaExcludes.scala | 8 +-------
project/SparkBuild.scala | 2 +-
.../apache/spark/tools/GenerateMIMAIgnore.scala | 14 ++++++++++----
5 files changed, 26 insertions(+), 12 deletions(-)
diff --git a/dev/mima b/dev/mima
index f9b9b03538..40603166c2 100755
--- a/dev/mima
+++ b/dev/mima
@@ -25,11 +25,19 @@ FWDIR="$(cd "`dirname "$0"`"/..; pwd)"
cd "$FWDIR"
echo -e "q\n" | sbt/sbt oldDeps/update
+rm -f .generated-mima*
+
+# Generate Mima Ignore is called twice, first with latest built jars
+# on the classpath and then again with previous version jars on the classpath.
+# Because of a bug in GenerateMIMAIgnore that when old jars are ahead on classpath
+# it did not process the new classes (which are in assembly jar).
+./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore
export SPARK_CLASSPATH="`find lib_managed \( -name '*spark*jar' -a -type f \) | tr "\\n" ":"`"
echo "SPARK_CLASSPATH=$SPARK_CLASSPATH"
./bin/spark-class org.apache.spark.tools.GenerateMIMAIgnore
+
echo -e "q\n" | sbt/sbt mima-report-binary-issues | grep -v -e "info.*Resolving"
ret_val=$?
diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index 0f5d71afcf..39f8ba4745 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -30,6 +30,12 @@ object MimaBuild {
def excludeMember(fullName: String) = Seq(
ProblemFilters.exclude[MissingMethodProblem](fullName),
+ // Sometimes excluded methods have default arguments and
+ // they are translated into public methods/fields($default$) in generated
+ // bytecode. It is not possible to exhustively list everything.
+ // But this should be okay.
+ ProblemFilters.exclude[MissingMethodProblem](fullName+"$default$2"),
+ ProblemFilters.exclude[MissingMethodProblem](fullName+"$default$1"),
ProblemFilters.exclude[MissingFieldProblem](fullName),
ProblemFilters.exclude[IncompatibleResultTypeProblem](fullName),
ProblemFilters.exclude[IncompatibleMethTypeProblem](fullName),
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 46b78bd5c7..2f1e05dfcc 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -37,14 +37,8 @@ object MimaExcludes {
Seq(
MimaBuild.excludeSparkPackage("deploy"),
MimaBuild.excludeSparkPackage("graphx")
- ) ++
- // This is @DeveloperAPI, but Mima still gives false-positives:
- MimaBuild.excludeSparkClass("scheduler.SparkListenerApplicationStart") ++
- Seq(
- // This is @Experimental, but Mima still gives false-positives:
- ProblemFilters.exclude[MissingMethodProblem](
- "org.apache.spark.api.java.JavaRDDLike.foreachAsync")
)
+
case v if v.startsWith("1.1") =>
Seq(
MimaBuild.excludeSparkPackage("deploy"),
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index c07ea313f1..ab9f8ba120 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -187,7 +187,7 @@ object OldDeps {
Some("org.apache.spark" % fullId % "1.1.0")
}
- def oldDepsSettings() = Defaults.defaultSettings ++ Seq(
+ def oldDepsSettings() = Defaults.coreDefaultSettings ++ Seq(
name := "old-deps",
scalaVersion := "2.10.4",
retrieveManaged := true,
diff --git a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
index bcf6d43ab3..595ded6ae6 100644
--- a/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
+++ b/tools/src/main/scala/org/apache/spark/tools/GenerateMIMAIgnore.scala
@@ -24,6 +24,7 @@ import scala.collection.mutable
import scala.collection.JavaConversions._
import scala.reflect.runtime.universe.runtimeMirror
import scala.reflect.runtime.{universe => unv}
+import scala.util.Try
/**
* A tool for generating classes to be excluded during binary checking with MIMA. It is expected
@@ -121,12 +122,17 @@ object GenerateMIMAIgnore {
}
def main(args: Array[String]) {
+ import scala.tools.nsc.io.File
val (privateClasses, privateMembers) = privateWithin("org.apache.spark")
- scala.tools.nsc.io.File(".generated-mima-class-excludes").
- writeAll(privateClasses.mkString("\n"))
+ val previousContents = Try(File(".generated-mima-class-excludes").lines()).
+ getOrElse(Iterator.empty).mkString("\n")
+ File(".generated-mima-class-excludes")
+ .writeAll(previousContents + privateClasses.mkString("\n"))
println("Created : .generated-mima-class-excludes in current directory.")
- scala.tools.nsc.io.File(".generated-mima-member-excludes").
- writeAll(privateMembers.mkString("\n"))
+ val previousMembersContents = Try(File(".generated-mima-member-excludes").lines)
+ .getOrElse(Iterator.empty).mkString("\n")
+ File(".generated-mima-member-excludes").writeAll(previousMembersContents +
+ privateMembers.mkString("\n"))
println("Created : .generated-mima-member-excludes in current directory.")
}
--
GitLab