diff --git a/assembly/pom.xml b/assembly/pom.xml
index 0c60b66c3daca2d971a54b913a8afa8000654adf..4f6aade133db71bd2b78e1f65b1b195ad8652465 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -32,6 +32,7 @@
<packaging>pom</packaging>
<properties>
+ <sbt.project.name>assembly</sbt.project.name>
<spark.jar.dir>scala-${scala.binary.version}</spark.jar.dir>
<spark.jar.basename>spark-assembly-${project.version}-hadoop${hadoop.version}.jar</spark.jar.basename>
<spark.jar>${project.build.directory}/${spark.jar.dir}/${spark.jar.basename}</spark.jar>
diff --git a/bagel/pom.xml b/bagel/pom.xml
index c8e39a415af2890eb90d61e38ef5c94afa5d9555..90c4b095bb6113e79c7de2b80e9f75e65ae8b0bb 100644
--- a/bagel/pom.xml
+++ b/bagel/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-bagel_2.10</artifactId>
+ <properties>
+ <sbt.project.name>bagel</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project Bagel</name>
<url>http://spark.apache.org/</url>
diff --git a/bin/spark-class b/bin/spark-class
index 04fa52c6756b1540a80181041530356569d9d18f..3f6beca5becf015537d605f93552d36f9797267d 100755
--- a/bin/spark-class
+++ b/bin/spark-class
@@ -110,9 +110,9 @@ export JAVA_OPTS
TOOLS_DIR="$FWDIR"/tools
SPARK_TOOLS_JAR=""
-if [ -e "$TOOLS_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar ]; then
+if [ -e "$TOOLS_DIR"/target/scala-$SCALA_VERSION/spark-tools*[0-9Tg].jar ]; then
# Use the JAR from the SBT build
- export SPARK_TOOLS_JAR=`ls "$TOOLS_DIR"/target/scala-$SCALA_VERSION/*assembly*[0-9Tg].jar`
+ export SPARK_TOOLS_JAR=`ls "$TOOLS_DIR"/target/scala-$SCALA_VERSION/spark-tools*[0-9Tg].jar`
fi
if [ -e "$TOOLS_DIR"/target/spark-tools*[0-9Tg].jar ]; then
# Use the JAR from the Maven build
diff --git a/core/pom.xml b/core/pom.xml
index 6abf8480d5da03ab3e2c8a9250cc4c7818734139..4ed920a750fffe02351d75b95fe54be177eb3961 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_2.10</artifactId>
+ <properties>
+ <sbt.project.name>core</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project Core</name>
<url>http://spark.apache.org/</url>
diff --git a/dev/run-tests b/dev/run-tests
index d9df020f7563c88b24ee2914f0963c860d8c1382..edd17b53b3d8cb262716514c7431f606451c5141 100755
--- a/dev/run-tests
+++ b/dev/run-tests
@@ -66,10 +66,10 @@ echo "========================================================================="
# (either resolution or compilation) prompts the user for input either q, r,
# etc to quit or retry. This echo is there to make it not block.
if [ -n "$_RUN_SQL_TESTS" ]; then
- echo -e "q\n" | SPARK_HIVE=true sbt/sbt clean assembly test | \
+ echo -e "q\n" | SPARK_HIVE=true sbt/sbt clean package assembly/assembly test | \
grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
else
- echo -e "q\n" | sbt/sbt clean assembly test | \
+ echo -e "q\n" | sbt/sbt clean package assembly/assembly test | \
grep -v -e "info.*Resolving" -e "warn.*Merging" -e "info.*Including"
fi
diff --git a/examples/pom.xml b/examples/pom.xml
index 4f6d7fdb87d479487cb7f274e3cc447df5d803ee..bd1c387c2eb91d24c16969e120879a51af5df513 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-examples_2.10</artifactId>
+ <properties>
+ <sbt.project.name>examples</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project Examples</name>
<url>http://spark.apache.org/</url>
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index c1f581967777beba3f19e92beebbc37711007bdf..61a6aff543aed262f8ef3a0491e67e4afb970b6d 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-flume_2.10</artifactId>
+ <properties>
+ <sbt.project.name>streaming-flume</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project External Flume</name>
<url>http://spark.apache.org/</url>
diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml
index d014a7aad0fca4ae1db8811720e618674b59130d..4762c50685a93dbd0a5ab6ece847c18bf4fdb55c 100644
--- a/external/kafka/pom.xml
+++ b/external/kafka/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-kafka_2.10</artifactId>
+ <properties>
+ <sbt.project.name>streaming-kafka</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project External Kafka</name>
<url>http://spark.apache.org/</url>
diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml
index 4980208cba3b041b42e4acac688539add1a13430..32c530e600ce0fd6e9620c706a1a0acec2e7ac3a 100644
--- a/external/mqtt/pom.xml
+++ b/external/mqtt/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-mqtt_2.10</artifactId>
+ <properties>
+ <sbt.project.name>streaming-mqtt</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project External MQTT</name>
<url>http://spark.apache.org/</url>
diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml
index 7073bd4404d9cea13a4bb39ef971a34ff69abcd3..637adb0f00da0c44ecb8cb0bd2de57cce518ed7f 100644
--- a/external/twitter/pom.xml
+++ b/external/twitter/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-twitter_2.10</artifactId>
+ <properties>
+ <sbt.project.name>streaming-twitter</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project External Twitter</name>
<url>http://spark.apache.org/</url>
diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml
index cf306e0dca8bd17ebc4efeec97a1bd86af7b6729..e4d758a04a4cd83a0b5038aa7e7bb2e85e3fb2b1 100644
--- a/external/zeromq/pom.xml
+++ b/external/zeromq/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming-zeromq_2.10</artifactId>
+ <properties>
+ <sbt.project.name>streaming-zeromq</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project External ZeroMQ</name>
<url>http://spark.apache.org/</url>
diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml
index 955ec1a8c30332df72c9d62e617f007fcc96fa60..3eade411b38b77daf711c36e7fa4bce74b57e640 100644
--- a/extras/java8-tests/pom.xml
+++ b/extras/java8-tests/pom.xml
@@ -28,7 +28,11 @@
<artifactId>java8-tests_2.10</artifactId>
<packaging>pom</packaging>
<name>Spark Project Java8 Tests POM</name>
-
+
+ <properties>
+ <sbt.project.name>java8-tests</sbt.project.name>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/extras/spark-ganglia-lgpl/pom.xml b/extras/spark-ganglia-lgpl/pom.xml
index 22ea330b4374debf79f20e0024688514ba6eeb66..a5b162a0482e4b7ab0168b2d7815eaff91898e0f 100644
--- a/extras/spark-ganglia-lgpl/pom.xml
+++ b/extras/spark-ganglia-lgpl/pom.xml
@@ -29,7 +29,11 @@
<artifactId>spark-ganglia-lgpl_2.10</artifactId>
<packaging>jar</packaging>
<name>Spark Ganglia Integration</name>
-
+
+ <properties>
+ <sbt.project.name>ganglia-lgpl</sbt.project.name>
+ </properties>
+
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 7d5d83e7f3bb9c6351c4a154e75b57bd147ffa28..7e3bcf29dcfbc75d29091cd1194d85945013a56a 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-graphx_2.10</artifactId>
+ <properties>
+ <sbt.project.name>graphx</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project GraphX</name>
<url>http://spark.apache.org/</url>
diff --git a/mllib/pom.xml b/mllib/pom.xml
index b622f96dd7901f3c9dceac4722847d6a65e81f58..87afd7ecf2dd493c10b2401de61fd0690d2ae177 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_2.10</artifactId>
+ <properties>
+ <sbt.project.name>mllib</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project ML Library</name>
<url>http://spark.apache.org/</url>
diff --git a/pom.xml b/pom.xml
index 05f76d566e9d1622a3cafcf008ad083452f3f621..fa80707d0929cb59402b381aa68706fe8cbb3cb4 100644
--- a/pom.xml
+++ b/pom.xml
@@ -110,7 +110,7 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>1.6</java.version>
-
+ <sbt.project.name>spark</sbt.project.name>
<scala.version>2.10.4</scala.version>
<scala.binary.version>2.10</scala.binary.version>
<mesos.version>0.18.1</mesos.version>
@@ -535,6 +535,10 @@
<groupId>org.mortbay.jetty</groupId>
<artifactId>servlet-api-2.5</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
@@ -618,6 +622,10 @@
<artifactId>hadoop-yarn-api</artifactId>
<version>${yarn.version}</version>
<exclusions>
+ <exclusion>
+ <groupId>javax.servlet</groupId>
+ <artifactId>servlet-api</artifactId>
+ </exclusion>
<exclusion>
<groupId>asm</groupId>
<artifactId>asm</artifactId>
diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index bb2d73741c3bfe8d6076fa325e97be593d0292d8..034ba6a7bf50f37e30750aa54295ac4d6be9f7be 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -15,13 +15,16 @@
* limitations under the License.
*/
+import sbt._
+import sbt.Keys.version
+
import com.typesafe.tools.mima.core._
import com.typesafe.tools.mima.core.MissingClassProblem
import com.typesafe.tools.mima.core.MissingTypesProblem
import com.typesafe.tools.mima.core.ProblemFilters._
import com.typesafe.tools.mima.plugin.MimaKeys.{binaryIssueFilters, previousArtifact}
import com.typesafe.tools.mima.plugin.MimaPlugin.mimaDefaultSettings
-import sbt._
+
object MimaBuild {
@@ -53,7 +56,7 @@ object MimaBuild {
excludePackage("org.apache.spark." + packageName)
}
- def ignoredABIProblems(base: File) = {
+ def ignoredABIProblems(base: File, currentSparkVersion: String) = {
// Excludes placed here will be used for all Spark versions
val defaultExcludes = Seq()
@@ -77,11 +80,16 @@ object MimaBuild {
}
defaultExcludes ++ ignoredClasses.flatMap(excludeClass) ++
- ignoredMembers.flatMap(excludeMember) ++ MimaExcludes.excludes
+ ignoredMembers.flatMap(excludeMember) ++ MimaExcludes.excludes(currentSparkVersion)
+ }
+
+ def mimaSettings(sparkHome: File, projectRef: ProjectRef) = {
+ val organization = "org.apache.spark"
+ val previousSparkVersion = "1.0.0"
+ val fullId = "spark-" + projectRef.project + "_2.10"
+ mimaDefaultSettings ++
+ Seq(previousArtifact := Some(organization % fullId % previousSparkVersion),
+ binaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value))
}
- def mimaSettings(sparkHome: File) = mimaDefaultSettings ++ Seq(
- previousArtifact := None,
- binaryIssueFilters ++= ignoredABIProblems(sparkHome)
- )
}
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index 1621833e124f5e54f7a8032e25c0c9ebf39bdafc..44bc9dc5fb690ae253e6ca8d7451f3f3187be73f 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -31,8 +31,8 @@ import com.typesafe.tools.mima.core._
* MimaBuild.excludeSparkClass("graphx.util.collection.GraphXPrimitiveKeyOpenHashMap")
*/
object MimaExcludes {
- val excludes =
- SparkBuild.SPARK_VERSION match {
+ def excludes(version: String) =
+ version match {
case v if v.startsWith("1.1") =>
Seq(
MimaBuild.excludeSparkPackage("deploy"),
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 599714233c18f01d7fcbe4c8c1fb1969810947e7..b55c50560bb936b608dfb6ccec529c754e0b9df9 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -15,524 +15,159 @@
* limitations under the License.
*/
-import sbt._
-import sbt.Classpaths.publishTask
-import sbt.Keys._
-import sbtassembly.Plugin._
-import AssemblyKeys._
import scala.util.Properties
-import org.scalastyle.sbt.ScalastylePlugin.{Settings => ScalaStyleSettings}
-import com.typesafe.tools.mima.plugin.MimaKeys.previousArtifact
-import sbtunidoc.Plugin._
-import UnidocKeys._
-
import scala.collection.JavaConversions._
-// For Sonatype publishing
-// import com.jsuereth.pgp.sbtplugin.PgpKeys._
-
-object SparkBuild extends Build {
- val SPARK_VERSION = "1.1.0-SNAPSHOT"
- val SPARK_VERSION_SHORT = SPARK_VERSION.replaceAll("-SNAPSHOT", "")
-
- // Hadoop version to build against. For example, "1.0.4" for Apache releases, or
- // "2.0.0-mr1-cdh4.2.0" for Cloudera Hadoop. Note that these variables can be set
- // through the environment variables SPARK_HADOOP_VERSION and SPARK_YARN.
- val DEFAULT_HADOOP_VERSION = "1.0.4"
-
- // Whether the Hadoop version to build against is 2.2.x, or a variant of it. This can be set
- // through the SPARK_IS_NEW_HADOOP environment variable.
- val DEFAULT_IS_NEW_HADOOP = false
-
- val DEFAULT_YARN = false
-
- val DEFAULT_HIVE = false
-
- // HBase version; set as appropriate.
- val HBASE_VERSION = "0.94.6"
-
- // Target JVM version
- val SCALAC_JVM_VERSION = "jvm-1.6"
- val JAVAC_JVM_VERSION = "1.6"
-
- lazy val root = Project("root", file("."), settings = rootSettings) aggregate(allProjects: _*)
-
- lazy val core = Project("core", file("core"), settings = coreSettings)
-
- /** Following project only exists to pull previous artifacts of Spark for generating
- Mima ignores. For more information see: SPARK 2071 */
- lazy val oldDeps = Project("oldDeps", file("dev"), settings = oldDepsSettings)
-
- def replDependencies = Seq[ProjectReference](core, graphx, bagel, mllib, sql) ++ maybeHiveRef
-
- lazy val repl = Project("repl", file("repl"), settings = replSettings)
- .dependsOn(replDependencies.map(a => a: sbt.ClasspathDep[sbt.ProjectReference]): _*)
-
- lazy val tools = Project("tools", file("tools"), settings = toolsSettings) dependsOn(core) dependsOn(streaming)
+import sbt._
+import sbt.Keys._
+import org.scalastyle.sbt.ScalastylePlugin.{Settings => ScalaStyleSettings}
+import com.typesafe.sbt.pom.{PomBuild, SbtPomKeys}
+import net.virtualvoid.sbt.graph.Plugin.graphSettings
- lazy val bagel = Project("bagel", file("bagel"), settings = bagelSettings) dependsOn(core)
+object BuildCommons {
- lazy val graphx = Project("graphx", file("graphx"), settings = graphxSettings) dependsOn(core)
+ private val buildLocation = file(".").getAbsoluteFile.getParentFile
- lazy val catalyst = Project("catalyst", file("sql/catalyst"), settings = catalystSettings) dependsOn(core)
+ val allProjects@Seq(bagel, catalyst, core, graphx, hive, mllib, repl, spark, sql, streaming,
+ streamingFlume, streamingKafka, streamingMqtt, streamingTwitter, streamingZeromq) =
+ Seq("bagel", "catalyst", "core", "graphx", "hive", "mllib", "repl", "spark", "sql",
+ "streaming", "streaming-flume", "streaming-kafka", "streaming-mqtt", "streaming-twitter",
+ "streaming-zeromq").map(ProjectRef(buildLocation, _))
- lazy val sql = Project("sql", file("sql/core"), settings = sqlCoreSettings) dependsOn(core) dependsOn(catalyst % "compile->compile;test->test")
+ val optionallyEnabledProjects@Seq(yarn, yarnStable, yarnAlpha, java8Tests, sparkGangliaLgpl) =
+ Seq("yarn", "yarn-stable", "yarn-alpha", "java8-tests", "ganglia-lgpl")
+ .map(ProjectRef(buildLocation, _))
- lazy val hive = Project("hive", file("sql/hive"), settings = hiveSettings) dependsOn(sql)
+ val assemblyProjects@Seq(assembly, examples) = Seq("assembly", "examples")
+ .map(ProjectRef(buildLocation, _))
- lazy val maybeHive: Seq[ClasspathDependency] = if (isHiveEnabled) Seq(hive) else Seq()
- lazy val maybeHiveRef: Seq[ProjectReference] = if (isHiveEnabled) Seq(hive) else Seq()
+ val tools = "tools"
- lazy val streaming = Project("streaming", file("streaming"), settings = streamingSettings) dependsOn(core)
+ val sparkHome = buildLocation
+}
- lazy val mllib = Project("mllib", file("mllib"), settings = mllibSettings) dependsOn(core)
+object SparkBuild extends PomBuild {
- lazy val assemblyProj = Project("assembly", file("assembly"), settings = assemblyProjSettings)
- .dependsOn(core, graphx, bagel, mllib, streaming, repl, sql) dependsOn(maybeYarn: _*) dependsOn(maybeHive: _*) dependsOn(maybeGanglia: _*)
+ import BuildCommons._
+ import scala.collection.mutable.Map
- lazy val assembleDepsTask = TaskKey[Unit]("assemble-deps")
- lazy val assembleDeps = assembleDepsTask := {
- println()
- println("**** NOTE ****")
- println("'sbt/sbt assemble-deps' is no longer supported.")
- println("Instead create a normal assembly and:")
- println(" export SPARK_PREPEND_CLASSES=1 (toggle on)")
- println(" unset SPARK_PREPEND_CLASSES (toggle off)")
- println()
- }
+ val projectsMap: Map[String, Seq[Setting[_]]] = Map.empty
- // A configuration to set an alternative publishLocalConfiguration
- lazy val MavenCompile = config("m2r") extend(Compile)
- lazy val publishLocalBoth = TaskKey[Unit]("publish-local", "publish local for m2 and ivy")
- val sparkHome = System.getProperty("user.dir")
-
- // Allows build configuration to be set through environment variables
- lazy val hadoopVersion = Properties.envOrElse("SPARK_HADOOP_VERSION", DEFAULT_HADOOP_VERSION)
- lazy val isNewHadoop = Properties.envOrNone("SPARK_IS_NEW_HADOOP") match {
- case None => {
- val isNewHadoopVersion = "^2\\.[2-9]+".r.findFirstIn(hadoopVersion).isDefined
- (isNewHadoopVersion|| DEFAULT_IS_NEW_HADOOP)
+ // Provides compatibility for older versions of the Spark build
+ def backwardCompatibility = {
+ import scala.collection.mutable
+ var isAlphaYarn = false
+ var profiles: mutable.Seq[String] = mutable.Seq.empty
+ if (Properties.envOrNone("SPARK_GANGLIA_LGPL").isDefined) {
+ println("NOTE: SPARK_GANGLIA_LGPL is deprecated, please use -Pganglia-lgpl flag.")
+ profiles ++= Seq("spark-ganglia-lgpl")
}
- case Some(v) => v.toBoolean
- }
-
- lazy val isYarnEnabled = Properties.envOrNone("SPARK_YARN") match {
- case None => DEFAULT_YARN
- case Some(v) => v.toBoolean
+ if (Properties.envOrNone("SPARK_HIVE").isDefined) {
+ println("NOTE: SPARK_HIVE is deprecated, please use -Phive flag.")
+ profiles ++= Seq("hive")
+ }
+ Properties.envOrNone("SPARK_HADOOP_VERSION") match {
+ case Some(v) =>
+ if (v.matches("0.23.*")) isAlphaYarn = true
+ println("NOTE: SPARK_HADOOP_VERSION is deprecated, please use -Dhadoop.version=" + v)
+ System.setProperty("hadoop.version", v)
+ case None =>
+ }
+ if (Properties.envOrNone("SPARK_YARN").isDefined) {
+ if(isAlphaYarn) {
+ println("NOTE: SPARK_YARN is deprecated, please use -Pyarn-alpha flag.")
+ profiles ++= Seq("yarn-alpha")
+ }
+ else {
+ println("NOTE: SPARK_YARN is deprecated, please use -Pyarn flag.")
+ profiles ++= Seq("yarn")
+ }
+ }
+ profiles
}
- lazy val hadoopClient = if (hadoopVersion.startsWith("0.20.") || hadoopVersion == "1.0.0") "hadoop-core" else "hadoop-client"
- val maybeAvro = if (hadoopVersion.startsWith("0.23.")) Seq("org.apache.avro" % "avro" % "1.7.4") else Seq()
- lazy val isHiveEnabled = Properties.envOrNone("SPARK_HIVE") match {
- case None => DEFAULT_HIVE
- case Some(v) => v.toBoolean
+ override val profiles = Properties.envOrNone("MAVEN_PROFILES") match {
+ case None => backwardCompatibility
+ // Rationale: If -P option exists no need to support backwardCompatibility.
+ case Some(v) =>
+ if (backwardCompatibility.nonEmpty)
+ println("Note: We ignore environment variables, when use of profile is detected in " +
+ "conjunction with environment variable.")
+ v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
}
- // Include Ganglia integration if the user has enabled Ganglia
- // This is isolated from the normal build due to LGPL-licensed code in the library
- lazy val isGangliaEnabled = Properties.envOrNone("SPARK_GANGLIA_LGPL").isDefined
- lazy val gangliaProj = Project("spark-ganglia-lgpl", file("extras/spark-ganglia-lgpl"), settings = gangliaSettings).dependsOn(core)
- val maybeGanglia: Seq[ClasspathDependency] = if (isGangliaEnabled) Seq(gangliaProj) else Seq()
- val maybeGangliaRef: Seq[ProjectReference] = if (isGangliaEnabled) Seq(gangliaProj) else Seq()
+ override val userPropertiesMap = System.getProperties.toMap
- // Include the Java 8 project if the JVM version is 8+
- lazy val javaVersion = System.getProperty("java.specification.version")
- lazy val isJava8Enabled = javaVersion.toDouble >= "1.8".toDouble
- val maybeJava8Tests = if (isJava8Enabled) Seq[ProjectReference](java8Tests) else Seq[ProjectReference]()
- lazy val java8Tests = Project("java8-tests", file("extras/java8-tests"), settings = java8TestsSettings).
- dependsOn(core) dependsOn(streaming % "compile->compile;test->test")
-
- // Include the YARN project if the user has enabled YARN
- lazy val yarnAlpha = Project("yarn-alpha", file("yarn/alpha"), settings = yarnAlphaSettings) dependsOn(core)
- lazy val yarn = Project("yarn", file("yarn/stable"), settings = yarnSettings) dependsOn(core)
-
- lazy val maybeYarn: Seq[ClasspathDependency] = if (isYarnEnabled) Seq(if (isNewHadoop) yarn else yarnAlpha) else Seq()
- lazy val maybeYarnRef: Seq[ProjectReference] = if (isYarnEnabled) Seq(if (isNewHadoop) yarn else yarnAlpha) else Seq()
-
- lazy val externalTwitter = Project("external-twitter", file("external/twitter"), settings = twitterSettings)
- .dependsOn(streaming % "compile->compile;test->test")
-
- lazy val externalKafka = Project("external-kafka", file("external/kafka"), settings = kafkaSettings)
- .dependsOn(streaming % "compile->compile;test->test")
-
- lazy val externalFlume = Project("external-flume", file("external/flume"), settings = flumeSettings)
- .dependsOn(streaming % "compile->compile;test->test")
-
- lazy val externalZeromq = Project("external-zeromq", file("external/zeromq"), settings = zeromqSettings)
- .dependsOn(streaming % "compile->compile;test->test")
-
- lazy val externalMqtt = Project("external-mqtt", file("external/mqtt"), settings = mqttSettings)
- .dependsOn(streaming % "compile->compile;test->test")
-
- lazy val allExternal = Seq[ClasspathDependency](externalTwitter, externalKafka, externalFlume, externalZeromq, externalMqtt)
- lazy val allExternalRefs = Seq[ProjectReference](externalTwitter, externalKafka, externalFlume, externalZeromq, externalMqtt)
-
- lazy val examples = Project("examples", file("examples"), settings = examplesSettings)
- .dependsOn(core, mllib, graphx, bagel, streaming, hive) dependsOn(allExternal: _*)
-
- // Everything except assembly, hive, tools, java8Tests and examples belong to packageProjects
- lazy val packageProjects = Seq[ProjectReference](core, repl, bagel, streaming, mllib, graphx, catalyst, sql) ++ maybeYarnRef ++ maybeHiveRef ++ maybeGangliaRef
-
- lazy val allProjects = packageProjects ++ allExternalRefs ++
- Seq[ProjectReference](examples, tools, assemblyProj) ++ maybeJava8Tests
-
- def sharedSettings = Defaults.defaultSettings ++ MimaBuild.mimaSettings(file(sparkHome)) ++ Seq(
- organization := "org.apache.spark",
- version := SPARK_VERSION,
- scalaVersion := "2.10.4",
- scalacOptions := Seq("-Xmax-classfile-name", "120", "-unchecked", "-deprecation", "-feature",
- "-target:" + SCALAC_JVM_VERSION),
- javacOptions := Seq("-target", JAVAC_JVM_VERSION, "-source", JAVAC_JVM_VERSION),
- unmanagedJars in Compile <<= baseDirectory map { base => (base / "lib" ** "*.jar").classpath },
+ lazy val sharedSettings = graphSettings ++ ScalaStyleSettings ++ Seq (
+ javaHome := Properties.envOrNone("JAVA_HOME").map(file),
+ incOptions := incOptions.value.withNameHashing(true),
retrieveManaged := true,
- javaHome := Properties.envOrNone("JAVA_HOME").map(file),
- // This is to add convenience of enabling sbt -Dsbt.offline=true for making the build offline.
- offline := "true".equalsIgnoreCase(sys.props("sbt.offline")),
retrievePattern := "[type]s/[artifact](-[revision])(-[classifier]).[ext]",
- transitiveClassifiers in Scope.GlobalScope := Seq("sources"),
- testListeners <<= target.map(t => Seq(new eu.henkelmann.sbt.JUnitXmlTestsListener(t.getAbsolutePath))),
- incOptions := incOptions.value.withNameHashing(true),
- // Fork new JVMs for tests and set Java options for those
- fork := true,
- javaOptions in Test += "-Dspark.home=" + sparkHome,
- javaOptions in Test += "-Dspark.testing=1",
- javaOptions in Test += "-Dsun.io.serialization.extendedDebugInfo=true",
- javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark").map { case (k,v) => s"-D$k=$v" }.toSeq,
- javaOptions in Test ++= "-Xmx3g -XX:PermSize=128M -XX:MaxNewSize=256m -XX:MaxPermSize=1g".split(" ").toSeq,
- javaOptions += "-Xmx3g",
- // Show full stack trace and duration in test cases.
- testOptions in Test += Tests.Argument("-oDF"),
- // Remove certain packages from Scaladoc
- scalacOptions in (Compile, doc) := Seq(
- "-groups",
- "-skip-packages", Seq(
- "akka",
- "org.apache.spark.api.python",
- "org.apache.spark.network",
- "org.apache.spark.deploy",
- "org.apache.spark.util.collection"
- ).mkString(":"),
- "-doc-title", "Spark " + SPARK_VERSION_SHORT + " ScalaDoc"
- ),
-
- // Only allow one test at a time, even across projects, since they run in the same JVM
- concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),
-
- resolvers ++= Seq(
- // HTTPS is unavailable for Maven Central
- "Maven Repository" at "http://repo.maven.apache.org/maven2",
- "Apache Repository" at "https://repository.apache.org/content/repositories/releases",
- "JBoss Repository" at "https://repository.jboss.org/nexus/content/repositories/releases/",
- "MQTT Repository" at "https://repo.eclipse.org/content/repositories/paho-releases/",
- "Cloudera Repository" at "http://repository.cloudera.com/artifactory/cloudera-repos/",
- "Pivotal Repository" at "http://repo.spring.io/libs-release/",
- // For Sonatype publishing
- // "sonatype-snapshots" at "https://oss.sonatype.org/content/repositories/snapshots",
- // "sonatype-staging" at "https://oss.sonatype.org/service/local/staging/deploy/maven2/",
- // also check the local Maven repository ~/.m2
- Resolver.mavenLocal
- ),
-
- publishMavenStyle := true,
-
- // useGpg in Global := true,
-
- pomExtra := (
- <parent>
- <groupId>org.apache</groupId>
- <artifactId>apache</artifactId>
- <version>14</version>
- </parent>
- <url>http://spark.apache.org/</url>
- <licenses>
- <license>
- <name>Apache 2.0 License</name>
- <url>http://www.apache.org/licenses/LICENSE-2.0.html</url>
- <distribution>repo</distribution>
- </license>
- </licenses>
- <scm>
- <connection>scm:git:git@github.com:apache/spark.git</connection>
- <url>scm:git:git@github.com:apache/spark.git</url>
- </scm>
- <developers>
- <developer>
- <id>matei</id>
- <name>Matei Zaharia</name>
- <email>matei.zaharia@gmail.com</email>
- <url>http://www.cs.berkeley.edu/~matei</url>
- <organization>Apache Software Foundation</organization>
- <organizationUrl>http://spark.apache.org</organizationUrl>
- </developer>
- </developers>
- <issueManagement>
- <system>JIRA</system>
- <url>https://issues.apache.org/jira/browse/SPARK</url>
- </issueManagement>
- ),
-
- /*
- publishTo <<= version { (v: String) =>
- val nexus = "https://oss.sonatype.org/"
- if (v.trim.endsWith("SNAPSHOT"))
- Some("sonatype-snapshots" at nexus + "content/repositories/snapshots")
- else
- Some("sonatype-staging" at nexus + "service/local/staging/deploy/maven2")
- },
+ publishMavenStyle := true
+ )
- */
-
- libraryDependencies ++= Seq(
- "io.netty" % "netty-all" % "4.0.17.Final",
- "org.eclipse.jetty" % "jetty-server" % jettyVersion,
- "org.eclipse.jetty" % "jetty-util" % jettyVersion,
- "org.eclipse.jetty" % "jetty-plus" % jettyVersion,
- "org.eclipse.jetty" % "jetty-security" % jettyVersion,
- "org.scalatest" %% "scalatest" % "2.1.5" % "test",
- "org.scalacheck" %% "scalacheck" % "1.11.3" % "test",
- "com.novocode" % "junit-interface" % "0.10" % "test",
- "org.easymock" % "easymockclassextension" % "3.1" % "test",
- "org.mockito" % "mockito-all" % "1.9.0" % "test",
- "junit" % "junit" % "4.10" % "test",
- // Needed by cglib which is needed by easymock.
- "asm" % "asm" % "3.3.1" % "test"
- ),
+ /** Following project only exists to pull previous artifacts of Spark for generating
+ Mima ignores. For more information see: SPARK 2071 */
+ lazy val oldDeps = Project("oldDeps", file("dev"), settings = oldDepsSettings)
- testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
- parallelExecution := true,
- /* Workaround for issue #206 (fixed after SBT 0.11.0) */
- watchTransitiveSources <<= Defaults.inDependencies[Task[Seq[File]]](watchSources.task,
- const(std.TaskExtra.constant(Nil)), aggregate = true, includeRoot = true) apply { _.join.map(_.flatten) },
-
- otherResolvers := Seq(Resolver.file("dotM2", file(Path.userHome + "/.m2/repository"))),
- publishLocalConfiguration in MavenCompile <<= (packagedArtifacts, deliverLocal, ivyLoggingLevel) map {
- (arts, _, level) => new PublishConfiguration(None, "dotM2", arts, Seq(), level)
- },
- publishMavenStyle in MavenCompile := true,
- publishLocal in MavenCompile <<= publishTask(publishLocalConfiguration in MavenCompile, deliverLocal),
- publishLocalBoth <<= Seq(publishLocal in MavenCompile, publishLocal).dependOn
- ) ++ net.virtualvoid.sbt.graph.Plugin.graphSettings ++ ScalaStyleSettings ++ genjavadocSettings
-
- val akkaVersion = "2.2.3-shaded-protobuf"
- val chillVersion = "0.3.6"
- val codahaleMetricsVersion = "3.0.0"
- val jblasVersion = "1.2.3"
- val jets3tVersion = if ("^2\\.[3-9]+".r.findFirstIn(hadoopVersion).isDefined) "0.9.0" else "0.7.1"
- val jettyVersion = "8.1.14.v20131031"
- val hiveVersion = "0.12.0"
- val parquetVersion = "1.4.3"
- val slf4jVersion = "1.7.5"
-
- val excludeJBossNetty = ExclusionRule(organization = "org.jboss.netty")
- val excludeIONetty = ExclusionRule(organization = "io.netty")
- val excludeEclipseJetty = ExclusionRule(organization = "org.eclipse.jetty")
- val excludeAsm = ExclusionRule(organization = "org.ow2.asm")
- val excludeOldAsm = ExclusionRule(organization = "asm")
- val excludeCommonsLogging = ExclusionRule(organization = "commons-logging")
- val excludeSLF4J = ExclusionRule(organization = "org.slf4j")
- val excludeScalap = ExclusionRule(organization = "org.scala-lang", artifact = "scalap")
- val excludeHadoop = ExclusionRule(organization = "org.apache.hadoop")
- val excludeCurator = ExclusionRule(organization = "org.apache.curator")
- val excludePowermock = ExclusionRule(organization = "org.powermock")
- val excludeFastutil = ExclusionRule(organization = "it.unimi.dsi")
- val excludeJruby = ExclusionRule(organization = "org.jruby")
- val excludeThrift = ExclusionRule(organization = "org.apache.thrift")
- val excludeServletApi = ExclusionRule(organization = "javax.servlet", artifact = "servlet-api")
- val excludeJUnit = ExclusionRule(organization = "junit")
-
- def sparkPreviousArtifact(id: String, organization: String = "org.apache.spark",
- version: String = "1.0.0", crossVersion: String = "2.10"): Option[sbt.ModuleID] = {
- val fullId = if (crossVersion.isEmpty) id else id + "_" + crossVersion
- Some(organization % fullId % version) // the artifact to compare binary compatibility with
+ def versionArtifact(id: String): Option[sbt.ModuleID] = {
+ val fullId = id + "_2.10"
+ Some("org.apache.spark" % fullId % "1.0.0")
}
- def coreSettings = sharedSettings ++ Seq(
- name := "spark-core",
- libraryDependencies ++= Seq(
- "com.google.guava" % "guava" % "14.0.1",
- "org.apache.commons" % "commons-lang3" % "3.3.2",
- "org.apache.commons" % "commons-math3" % "3.3" % "test",
- "com.google.code.findbugs" % "jsr305" % "1.3.9",
- "log4j" % "log4j" % "1.2.17",
- "org.slf4j" % "slf4j-api" % slf4jVersion,
- "org.slf4j" % "slf4j-log4j12" % slf4jVersion,
- "org.slf4j" % "jul-to-slf4j" % slf4jVersion,
- "org.slf4j" % "jcl-over-slf4j" % slf4jVersion,
- "commons-daemon" % "commons-daemon" % "1.0.10", // workaround for bug HADOOP-9407
- "com.ning" % "compress-lzf" % "1.0.0",
- "org.xerial.snappy" % "snappy-java" % "1.0.5",
- "org.spark-project.akka" %% "akka-remote" % akkaVersion,
- "org.spark-project.akka" %% "akka-slf4j" % akkaVersion,
- "org.spark-project.akka" %% "akka-testkit" % akkaVersion % "test",
- "org.json4s" %% "json4s-jackson" % "3.2.6" excludeAll(excludeScalap),
- "colt" % "colt" % "1.2.0",
- "org.apache.mesos" % "mesos" % "0.18.1" classifier("shaded-protobuf") exclude("com.google.protobuf", "protobuf-java"),
- "commons-net" % "commons-net" % "2.2",
- "net.java.dev.jets3t" % "jets3t" % jets3tVersion excludeAll(excludeCommonsLogging),
- "commons-codec" % "commons-codec" % "1.5", // Prevent jets3t from including the older version of commons-codec
- "org.apache.derby" % "derby" % "10.4.2.0" % "test",
- "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeJBossNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm, excludeServletApi),
- "org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeJBossNetty),
- "com.codahale.metrics" % "metrics-core" % codahaleMetricsVersion,
- "com.codahale.metrics" % "metrics-jvm" % codahaleMetricsVersion,
- "com.codahale.metrics" % "metrics-json" % codahaleMetricsVersion,
- "com.codahale.metrics" % "metrics-graphite" % codahaleMetricsVersion,
- "com.twitter" %% "chill" % chillVersion excludeAll(excludeAsm),
- "com.twitter" % "chill-java" % chillVersion excludeAll(excludeAsm),
- "org.tachyonproject" % "tachyon" % "0.4.1-thrift" excludeAll(excludeHadoop, excludeCurator, excludeEclipseJetty, excludePowermock),
- "com.clearspring.analytics" % "stream" % "2.7.0" excludeAll(excludeFastutil), // Only HyperLogLogPlus is used, which does not depend on fastutil.
- "org.spark-project" % "pyrolite" % "2.0.1",
- "net.sf.py4j" % "py4j" % "0.8.1"
- ),
- libraryDependencies ++= maybeAvro,
- assembleDeps,
- previousArtifact := sparkPreviousArtifact("spark-core")
+ def oldDepsSettings() = Defaults.defaultSettings ++ Seq(
+ name := "old-deps",
+ scalaVersion := "2.10.4",
+ retrieveManaged := true,
+ retrievePattern := "[type]s/[artifact](-[revision])(-[classifier]).[ext]",
+ libraryDependencies := Seq("spark-streaming-mqtt", "spark-streaming-zeromq",
+ "spark-streaming-flume", "spark-streaming-kafka", "spark-streaming-twitter",
+ "spark-streaming", "spark-mllib", "spark-bagel", "spark-graphx",
+ "spark-core").map(versionArtifact(_).get intransitive())
)
- // Create a colon-separate package list adding "org.apache.spark" in front of all of them,
- // for easier specification of JavaDoc package groups
- def packageList(names: String*): String = {
- names.map(s => "org.apache.spark." + s).mkString(":")
+ def enable(settings: Seq[Setting[_]])(projectRef: ProjectRef) = {
+ val existingSettings = projectsMap.getOrElse(projectRef.project, Seq[Setting[_]]())
+ projectsMap += (projectRef.project -> (existingSettings ++ settings))
}
- def rootSettings = sharedSettings ++ scalaJavaUnidocSettings ++ Seq(
- publish := {},
+ // Note ordering of these settings matter.
+ /* Enable shared settings on all projects */
+ (allProjects ++ optionallyEnabledProjects ++ assemblyProjects).foreach(enable(sharedSettings))
- unidocProjectFilter in (ScalaUnidoc, unidoc) :=
- inAnyProject -- inProjects(repl, examples, tools, catalyst, yarn, yarnAlpha),
- unidocProjectFilter in (JavaUnidoc, unidoc) :=
- inAnyProject -- inProjects(repl, examples, bagel, graphx, catalyst, tools, yarn, yarnAlpha),
+ /* Enable tests settings for all projects except examples, assembly and tools */
+ (allProjects ++ optionallyEnabledProjects).foreach(enable(TestSettings.settings))
- // Skip class names containing $ and some internal packages in Javadocs
- unidocAllSources in (JavaUnidoc, unidoc) := {
- (unidocAllSources in (JavaUnidoc, unidoc)).value
- .map(_.filterNot(_.getName.contains("$")))
- .map(_.filterNot(_.getCanonicalPath.contains("akka")))
- .map(_.filterNot(_.getCanonicalPath.contains("deploy")))
- .map(_.filterNot(_.getCanonicalPath.contains("network")))
- .map(_.filterNot(_.getCanonicalPath.contains("executor")))
- .map(_.filterNot(_.getCanonicalPath.contains("python")))
- .map(_.filterNot(_.getCanonicalPath.contains("collection")))
- },
+ /* Enable Mima for all projects except spark, hive, catalyst, sql and repl */
+ // TODO: Add Sql to mima checks
+ allProjects.filterNot(y => Seq(spark, sql, hive, catalyst, repl).exists(x => x == y)).
+ foreach (x => enable(MimaBuild.mimaSettings(sparkHome, x))(x))
- // Javadoc options: create a window title, and group key packages on index page
- javacOptions in doc := Seq(
- "-windowtitle", "Spark " + SPARK_VERSION_SHORT + " JavaDoc",
- "-public",
- "-group", "Core Java API", packageList("api.java", "api.java.function"),
- "-group", "Spark Streaming", packageList(
- "streaming.api.java", "streaming.flume", "streaming.kafka",
- "streaming.mqtt", "streaming.twitter", "streaming.zeromq"
- ),
- "-group", "MLlib", packageList(
- "mllib.classification", "mllib.clustering", "mllib.evaluation.binary", "mllib.linalg",
- "mllib.linalg.distributed", "mllib.optimization", "mllib.rdd", "mllib.recommendation",
- "mllib.regression", "mllib.stat", "mllib.tree", "mllib.tree.configuration",
- "mllib.tree.impurity", "mllib.tree.model", "mllib.util"
- ),
- "-group", "Spark SQL", packageList("sql.api.java", "sql.hive.api.java"),
- "-noqualifier", "java.lang"
- )
- )
+ /* Enable Assembly for all assembly projects */
+ assemblyProjects.foreach(enable(Assembly.settings))
- def replSettings = sharedSettings ++ Seq(
- name := "spark-repl",
- libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-compiler" % v),
- libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "jline" % v),
- libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-reflect" % v)
- )
+ /* Enable unidoc only for the root spark project */
+ enable(Unidoc.settings)(spark)
- def examplesSettings = sharedSettings ++ Seq(
- name := "spark-examples",
- jarName in assembly <<= version map {
- v => "spark-examples-" + v + "-hadoop" + hadoopVersion + ".jar" },
- libraryDependencies ++= Seq(
- "com.twitter" %% "algebird-core" % "0.1.11",
- "org.apache.hbase" % "hbase" % HBASE_VERSION excludeAll(excludeIONetty, excludeJBossNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging, excludeJruby),
- "org.apache.cassandra" % "cassandra-all" % "1.2.6"
- exclude("com.google.guava", "guava")
- exclude("com.googlecode.concurrentlinkedhashmap", "concurrentlinkedhashmap-lru")
- exclude("com.ning","compress-lzf")
- exclude("io.netty", "netty")
- exclude("jline","jline")
- exclude("org.apache.cassandra.deps", "avro")
- excludeAll(excludeSLF4J, excludeIONetty),
- "com.github.scopt" %% "scopt" % "3.2.0"
- )
- ) ++ assemblySettings ++ extraAssemblySettings
-
- def toolsSettings = sharedSettings ++ Seq(
- name := "spark-tools",
- libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-compiler" % v),
- libraryDependencies <+= scalaVersion(v => "org.scala-lang" % "scala-reflect" % v )
- ) ++ assemblySettings ++ extraAssemblySettings
-
- def graphxSettings = sharedSettings ++ Seq(
- name := "spark-graphx",
- previousArtifact := sparkPreviousArtifact("spark-graphx"),
- libraryDependencies ++= Seq(
- "org.jblas" % "jblas" % jblasVersion
- )
- )
+ /* Hive console settings */
+ enable(Hive.settings)(hive)
- def bagelSettings = sharedSettings ++ Seq(
- name := "spark-bagel",
- previousArtifact := sparkPreviousArtifact("spark-bagel")
- )
+ // TODO: move this to its upstream project.
+ override def projectDefinitions(baseDirectory: File): Seq[Project] = {
+ super.projectDefinitions(baseDirectory).map { x =>
+ if (projectsMap.exists(_._1 == x.id)) x.settings(projectsMap(x.id): _*)
+ else x.settings(Seq[Setting[_]](): _*)
+ } ++ Seq[Project](oldDeps)
+ }
- def mllibSettings = sharedSettings ++ Seq(
- name := "spark-mllib",
- previousArtifact := sparkPreviousArtifact("spark-mllib"),
- libraryDependencies ++= Seq(
- "org.jblas" % "jblas" % jblasVersion,
- "org.scalanlp" %% "breeze" % "0.7" excludeAll(excludeJUnit)
- )
- )
+}
- def catalystSettings = sharedSettings ++ Seq(
- name := "catalyst",
- // The mechanics of rewriting expression ids to compare trees in some test cases makes
- // assumptions about the the expression ids being contiguous. Running tests in parallel breaks
- // this non-deterministically. TODO: FIX THIS.
- parallelExecution in Test := false,
- libraryDependencies ++= Seq(
- "com.typesafe" %% "scalalogging-slf4j" % "1.0.1"
- )
- )
+object Hive {
- def sqlCoreSettings = sharedSettings ++ Seq(
- name := "spark-sql",
- libraryDependencies ++= Seq(
- "com.twitter" % "parquet-column" % parquetVersion,
- "com.twitter" % "parquet-hadoop" % parquetVersion,
- "com.fasterxml.jackson.core" % "jackson-databind" % "2.3.0" // json4s-jackson 3.2.6 requires jackson-databind 2.3.0.
- ),
- initialCommands in console :=
- """
- |import org.apache.spark.sql.catalyst.analysis._
- |import org.apache.spark.sql.catalyst.dsl._
- |import org.apache.spark.sql.catalyst.errors._
- |import org.apache.spark.sql.catalyst.expressions._
- |import org.apache.spark.sql.catalyst.plans.logical._
- |import org.apache.spark.sql.catalyst.rules._
- |import org.apache.spark.sql.catalyst.types._
- |import org.apache.spark.sql.catalyst.util._
- |import org.apache.spark.sql.execution
- |import org.apache.spark.sql.test.TestSQLContext._
- |import org.apache.spark.sql.parquet.ParquetTestData""".stripMargin
- )
+ lazy val settings = Seq(
- // Since we don't include hive in the main assembly this project also acts as an alternative
- // assembly jar.
- def hiveSettings = sharedSettings ++ Seq(
- name := "spark-hive",
javaOptions += "-XX:MaxPermSize=1g",
- libraryDependencies ++= Seq(
- "org.spark-project.hive" % "hive-metastore" % hiveVersion,
- "org.spark-project.hive" % "hive-exec" % hiveVersion excludeAll(excludeCommonsLogging),
- "org.spark-project.hive" % "hive-serde" % hiveVersion
- ),
- // Multiple queries rely on the TestHive singleton. See comments there for more details.
+ // Multiple queries rely on the TestHive singleton. See comments there for more details.
parallelExecution in Test := false,
// Supporting all SerDes requires us to depend on deprecated APIs, so we turn off the warnings
// only for this subproject.
@@ -555,67 +190,16 @@ object SparkBuild extends Build {
|import org.apache.spark.sql.parquet.ParquetTestData""".stripMargin
)
- def streamingSettings = sharedSettings ++ Seq(
- name := "spark-streaming",
- previousArtifact := sparkPreviousArtifact("spark-streaming")
- )
-
- def yarnCommonSettings = sharedSettings ++ Seq(
- unmanagedSourceDirectories in Compile <++= baseDirectory { base =>
- Seq(
- base / "../common/src/main/scala"
- )
- },
-
- unmanagedSourceDirectories in Test <++= baseDirectory { base =>
- Seq(
- base / "../common/src/test/scala"
- )
- }
-
- ) ++ extraYarnSettings
-
- def yarnAlphaSettings = yarnCommonSettings ++ Seq(
- name := "spark-yarn-alpha"
- )
-
- def yarnSettings = yarnCommonSettings ++ Seq(
- name := "spark-yarn"
- )
-
- def gangliaSettings = sharedSettings ++ Seq(
- name := "spark-ganglia-lgpl",
- libraryDependencies += "com.codahale.metrics" % "metrics-ganglia" % "3.0.0"
- )
-
- def java8TestsSettings = sharedSettings ++ Seq(
- name := "java8-tests",
- javacOptions := Seq("-target", "1.8", "-source", "1.8"),
- testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a")
- )
-
- // Conditionally include the YARN dependencies because some tools look at all sub-projects and will complain
- // if we refer to nonexistent dependencies (e.g. hadoop-yarn-api from a Hadoop version without YARN).
- def extraYarnSettings = if(isYarnEnabled) yarnEnabledSettings else Seq()
-
- def yarnEnabledSettings = Seq(
- libraryDependencies ++= Seq(
- // Exclude rule required for all ?
- "org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeJBossNetty, excludeAsm, excludeOldAsm),
- "org.apache.hadoop" % "hadoop-yarn-api" % hadoopVersion excludeAll(excludeJBossNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging),
- "org.apache.hadoop" % "hadoop-yarn-common" % hadoopVersion excludeAll(excludeJBossNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging),
- "org.apache.hadoop" % "hadoop-yarn-client" % hadoopVersion excludeAll(excludeJBossNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging),
- "org.apache.hadoop" % "hadoop-yarn-server-web-proxy" % hadoopVersion excludeAll(excludeJBossNetty, excludeAsm, excludeOldAsm, excludeCommonsLogging, excludeServletApi)
- )
- )
+}
- def assemblyProjSettings = sharedSettings ++ Seq(
- name := "spark-assembly",
- jarName in assembly <<= version map { v => "spark-assembly-" + v + "-hadoop" + hadoopVersion + ".jar" }
- ) ++ assemblySettings ++ extraAssemblySettings
+object Assembly {
+ import sbtassembly.Plugin._
+ import AssemblyKeys._
- def extraAssemblySettings() = Seq(
+ lazy val settings = assemblySettings ++ Seq(
test in assembly := {},
+ jarName in assembly <<= (version, moduleName) map { (v, mName) => mName + "-"+v + "-hadoop" +
+ Option(System.getProperty("hadoop.version")).getOrElse("1.0.4") + ".jar" },
mergeStrategy in assembly := {
case PathList("org", "datanucleus", xs @ _*) => MergeStrategy.discard
case m if m.toLowerCase.endsWith("manifest.mf") => MergeStrategy.discard
@@ -627,57 +211,95 @@ object SparkBuild extends Build {
}
)
- def oldDepsSettings() = Defaults.defaultSettings ++ Seq(
- name := "old-deps",
- scalaVersion := "2.10.4",
- retrieveManaged := true,
- retrievePattern := "[type]s/[artifact](-[revision])(-[classifier]).[ext]",
- libraryDependencies := Seq("spark-streaming-mqtt", "spark-streaming-zeromq",
- "spark-streaming-flume", "spark-streaming-kafka", "spark-streaming-twitter",
- "spark-streaming", "spark-mllib", "spark-bagel", "spark-graphx",
- "spark-core").map(sparkPreviousArtifact(_).get intransitive())
- )
+}
- def twitterSettings() = sharedSettings ++ Seq(
- name := "spark-streaming-twitter",
- previousArtifact := sparkPreviousArtifact("spark-streaming-twitter"),
- libraryDependencies ++= Seq(
- "org.twitter4j" % "twitter4j-stream" % "3.0.3"
- )
- )
+object Unidoc {
- def kafkaSettings() = sharedSettings ++ Seq(
- name := "spark-streaming-kafka",
- previousArtifact := sparkPreviousArtifact("spark-streaming-kafka"),
- libraryDependencies ++= Seq(
- "com.github.sgroschupf" % "zkclient" % "0.1",
- "org.apache.kafka" %% "kafka" % "0.8.0"
- exclude("com.sun.jdmk", "jmxtools")
- exclude("com.sun.jmx", "jmxri")
- exclude("net.sf.jopt-simple", "jopt-simple")
- excludeAll(excludeSLF4J)
- )
- )
+ import BuildCommons._
+ import sbtunidoc.Plugin._
+ import UnidocKeys._
+
+ // for easier specification of JavaDoc package groups
+ private def packageList(names: String*): String = {
+ names.map(s => "org.apache.spark." + s).mkString(":")
+ }
- def flumeSettings() = sharedSettings ++ Seq(
- name := "spark-streaming-flume",
- previousArtifact := sparkPreviousArtifact("spark-streaming-flume"),
- libraryDependencies ++= Seq(
- "org.apache.flume" % "flume-ng-sdk" % "1.4.0" % "compile" excludeAll(excludeIONetty, excludeThrift)
+ lazy val settings = scalaJavaUnidocSettings ++ Seq (
+ publish := {},
+
+ unidocProjectFilter in(ScalaUnidoc, unidoc) :=
+ inAnyProject -- inProjects(repl, examples, tools, catalyst, yarn, yarnAlpha),
+ unidocProjectFilter in(JavaUnidoc, unidoc) :=
+ inAnyProject -- inProjects(repl, bagel, graphx, examples, tools, catalyst, yarn, yarnAlpha),
+
+ // Skip class names containing $ and some internal packages in Javadocs
+ unidocAllSources in (JavaUnidoc, unidoc) := {
+ (unidocAllSources in (JavaUnidoc, unidoc)).value
+ .map(_.filterNot(_.getName.contains("$")))
+ .map(_.filterNot(_.getCanonicalPath.contains("akka")))
+ .map(_.filterNot(_.getCanonicalPath.contains("deploy")))
+ .map(_.filterNot(_.getCanonicalPath.contains("network")))
+ .map(_.filterNot(_.getCanonicalPath.contains("executor")))
+ .map(_.filterNot(_.getCanonicalPath.contains("python")))
+ .map(_.filterNot(_.getCanonicalPath.contains("collection")))
+ },
+
+ // Javadoc options: create a window title, and group key packages on index page
+ javacOptions in doc := Seq(
+ "-windowtitle", "Spark " + version.value.replaceAll("-SNAPSHOT", "") + " JavaDoc",
+ "-public",
+ "-group", "Core Java API", packageList("api.java", "api.java.function"),
+ "-group", "Spark Streaming", packageList(
+ "streaming.api.java", "streaming.flume", "streaming.kafka",
+ "streaming.mqtt", "streaming.twitter", "streaming.zeromq"
+ ),
+ "-group", "MLlib", packageList(
+ "mllib.classification", "mllib.clustering", "mllib.evaluation.binary", "mllib.linalg",
+ "mllib.linalg.distributed", "mllib.optimization", "mllib.rdd", "mllib.recommendation",
+ "mllib.regression", "mllib.stat", "mllib.tree", "mllib.tree.configuration",
+ "mllib.tree.impurity", "mllib.tree.model", "mllib.util"
+ ),
+ "-group", "Spark SQL", packageList("sql.api.java", "sql.hive.api.java"),
+ "-noqualifier", "java.lang"
)
)
+}
- def zeromqSettings() = sharedSettings ++ Seq(
- name := "spark-streaming-zeromq",
- previousArtifact := sparkPreviousArtifact("spark-streaming-zeromq"),
- libraryDependencies ++= Seq(
- "org.spark-project.akka" %% "akka-zeromq" % akkaVersion
+object TestSettings {
+ import BuildCommons._
+
+ lazy val settings = Seq (
+ // Fork new JVMs for tests and set Java options for those
+ fork := true,
+ javaOptions in Test += "-Dspark.home=" + sparkHome,
+ javaOptions in Test += "-Dspark.testing=1",
+ javaOptions in Test += "-Dsun.io.serialization.extendedDebugInfo=true",
+ javaOptions in Test ++= System.getProperties.filter(_._1 startsWith "spark")
+ .map { case (k,v) => s"-D$k=$v" }.toSeq,
+ javaOptions in Test ++= "-Xmx3g -XX:PermSize=128M -XX:MaxNewSize=256m -XX:MaxPermSize=1g"
+ .split(" ").toSeq,
+ javaOptions += "-Xmx3g",
+
+ // Show full stack trace and duration in test cases.
+ testOptions in Test += Tests.Argument("-oDF"),
+ testOptions += Tests.Argument(TestFrameworks.JUnit, "-v", "-a"),
+ // Enable Junit testing.
+ libraryDependencies += "com.novocode" % "junit-interface" % "0.9" % "test",
+ // Only allow one test at a time, even across projects, since they run in the same JVM
+ parallelExecution in Test := false,
+ concurrentRestrictions in Global += Tags.limit(Tags.Test, 1),
+ // Remove certain packages from Scaladoc
+ scalacOptions in (Compile, doc) := Seq(
+ "-groups",
+ "-skip-packages", Seq(
+ "akka",
+ "org.apache.spark.api.python",
+ "org.apache.spark.network",
+ "org.apache.spark.deploy",
+ "org.apache.spark.util.collection"
+ ).mkString(":"),
+ "-doc-title", "Spark " + version.value.replaceAll("-SNAPSHOT", "") + " ScalaDoc"
)
)
- def mqttSettings() = streamingSettings ++ Seq(
- name := "spark-streaming-mqtt",
- previousArtifact := sparkPreviousArtifact("spark-streaming-mqtt"),
- libraryDependencies ++= Seq("org.eclipse.paho" % "mqtt-client" % "0.4.0")
- )
}
diff --git a/project/build.properties b/project/build.properties
index bcde13f4362a74ca3d6c9dba98d6c72563589968..c12ef652adfcb5c88e0d78984f1b000652e5b88e 100644
--- a/project/build.properties
+++ b/project/build.properties
@@ -14,4 +14,4 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
-sbt.version=0.13.2
+sbt.version=0.13.5
diff --git a/project/project/SparkPluginBuild.scala b/project/project/SparkPluginBuild.scala
index e9fba641eb8a1f617f0b214e3b6080ab1aa4374c..3ef2d5451da0de801107d18ab3db77decb1bba2b 100644
--- a/project/project/SparkPluginBuild.scala
+++ b/project/project/SparkPluginBuild.scala
@@ -24,8 +24,10 @@ import sbt.Keys._
* becomes available for scalastyle sbt plugin.
*/
object SparkPluginDef extends Build {
- lazy val root = Project("plugins", file(".")) dependsOn(sparkStyle)
+ lazy val root = Project("plugins", file(".")) dependsOn(sparkStyle, sbtPomReader)
lazy val sparkStyle = Project("spark-style", file("spark-style"), settings = styleSettings)
+ lazy val sbtPomReader = uri("https://github.com/ScrapCodes/sbt-pom-reader.git")
+
// There is actually no need to publish this artifact.
def styleSettings = Defaults.defaultSettings ++ Seq (
name := "spark-style",
diff --git a/repl/pom.xml b/repl/pom.xml
index 4a66408ef3d2d0c3f6d16c0ef3ec900e196e9124..4ebb1b82f0e8c557d7349c4fe21979dba596b45f 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -32,6 +32,7 @@
<url>http://spark.apache.org/</url>
<properties>
+ <sbt.project.name>repl</sbt.project.name>
<deb.install.path>/usr/share/spark</deb.install.path>
<deb.user>root</deb.user>
</properties>
diff --git a/sbt/sbt b/sbt/sbt
index 9de265bd07dcb4c281023b51450fd207235f54ab..1b1aa1483a829df6a2115f0917e3daed9992e4e8 100755
--- a/sbt/sbt
+++ b/sbt/sbt
@@ -72,6 +72,7 @@ Usage: $script_name [options]
-J-X pass option -X directly to the java runtime
(-J is stripped)
-S-X add -X to sbt's scalacOptions (-J is stripped)
+ -PmavenProfiles Enable a maven profile for the build.
In the case of duplicated or conflicting options, the order above
shows precedence: JAVA_OPTS lowest, command line options highest.
diff --git a/sbt/sbt-launch-lib.bash b/sbt/sbt-launch-lib.bash
index 64e40a88206bef296294184f606ce088e07d60d4..857b62ffa229c6b1b1dac709941588196f32cabe 100755
--- a/sbt/sbt-launch-lib.bash
+++ b/sbt/sbt-launch-lib.bash
@@ -16,6 +16,7 @@ declare -a residual_args
declare -a java_args
declare -a scalac_args
declare -a sbt_commands
+declare -a maven_profiles
if test -x "$JAVA_HOME/bin/java"; then
echo -e "Using $JAVA_HOME as default JAVA_HOME."
@@ -87,6 +88,13 @@ addJava () {
dlog "[addJava] arg = '$1'"
java_args=( "${java_args[@]}" "$1" )
}
+
+enableProfile () {
+ dlog "[enableProfile] arg = '$1'"
+ maven_profiles=( "${maven_profiles[@]}" "$1" )
+ export MAVEN_PROFILES="${maven_profiles[@]}"
+}
+
addSbt () {
dlog "[addSbt] arg = '$1'"
sbt_commands=( "${sbt_commands[@]}" "$1" )
@@ -141,7 +149,8 @@ process_args () {
-java-home) require_arg path "$1" "$2" && java_cmd="$2/bin/java" && export JAVA_HOME=$2 && shift 2 ;;
-D*) addJava "$1" && shift ;;
- -J*) addJava "${1:2}" && shift ;;
+ -J*) addJava "${1:2}" && shift ;;
+ -P*) enableProfile "$1" && shift ;;
*) addResidual "$1" && shift ;;
esac
done
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 01d7b569080ead26133c43297297bc70f50c4405..6decde3fcd62d909ef576ffcfda5204eaecbdf87 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -31,6 +31,9 @@
<packaging>jar</packaging>
<name>Spark Project Catalyst</name>
<url>http://spark.apache.org/</url>
+ <properties>
+ <sbt.project.name>catalyst</sbt.project.name>
+ </properties>
<dependencies>
<dependency>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 8210fd1f210d1cc83bbae52038ca566c36ca5471..c309c43804d972388f609c433caec14b1ca8b9f0 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -31,6 +31,9 @@
<packaging>jar</packaging>
<name>Spark Project SQL</name>
<url>http://spark.apache.org/</url>
+ <properties>
+ <sbt.project.name>sql</sbt.project.name>
+ </properties>
<dependencies>
<dependency>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 5ede76e5c3904d5b3d90c2fa0cda28c742d784c9..f30ae28b81e0634da1c92cd55e33f3be9d010adf 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -31,6 +31,9 @@
<packaging>jar</packaging>
<name>Spark Project Hive</name>
<url>http://spark.apache.org/</url>
+ <properties>
+ <sbt.project.name>hive</sbt.project.name>
+ </properties>
<dependencies>
<dependency>
@@ -48,6 +51,11 @@
<artifactId>hive-metastore</artifactId>
<version>${hive.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-httpclient</groupId>
+ <artifactId>commons-httpclient</artifactId>
+ <version>3.1</version>
+ </dependency>
<dependency>
<groupId>org.spark-project.hive</groupId>
<artifactId>hive-exec</artifactId>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index f506d6ce34a6f0e6f78bb9c70c344d1500646d1d..f60697ce745b781dbf23e938cf5981f07b841785 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -27,6 +27,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_2.10</artifactId>
+ <properties>
+ <sbt.project.name>streaming</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project Streaming</name>
<url>http://spark.apache.org/</url>
diff --git a/tools/pom.xml b/tools/pom.xml
index 79cd8551d07221acf64cda4da0475a7ee445b895..c0ee8faa7a61526f36731197afa5f782fc0f7bfc 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -26,6 +26,9 @@
<groupId>org.apache.spark</groupId>
<artifactId>spark-tools_2.10</artifactId>
+ <properties>
+ <sbt.project.name>tools</sbt.project.name>
+ </properties>
<packaging>jar</packaging>
<name>Spark Project Tools</name>
<url>http://spark.apache.org/</url>
diff --git a/yarn/alpha/pom.xml b/yarn/alpha/pom.xml
index b8a631dd0bb3b842734d7c198cd1de32b09d8f03..5b13a1f002d6e3550f9fc69c11583648eaf0b0f6 100644
--- a/yarn/alpha/pom.xml
+++ b/yarn/alpha/pom.xml
@@ -23,6 +23,9 @@
<version>1.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
+ <properties>
+ <sbt.project.name>yarn-alpha</sbt.project.name>
+ </properties>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn-alpha_2.10</artifactId>
diff --git a/yarn/pom.xml b/yarn/pom.xml
index ef7066ef1fdfc406d6fc96d7f651c1fd6f13cd91..efb473aa1b2614c1711ad7124011b664e01804ae 100644
--- a/yarn/pom.xml
+++ b/yarn/pom.xml
@@ -28,6 +28,9 @@
<artifactId>yarn-parent_2.10</artifactId>
<packaging>pom</packaging>
<name>Spark Project YARN Parent POM</name>
+ <properties>
+ <sbt.project.name>yarn</sbt.project.name>
+ </properties>
<dependencies>
<dependency>
diff --git a/yarn/stable/pom.xml b/yarn/stable/pom.xml
index 0931beb50550889f8432c50f2db47277925a888d..ceaf9f9d71001e2f2a490a513fd04d3d01d4abf1 100644
--- a/yarn/stable/pom.xml
+++ b/yarn/stable/pom.xml
@@ -23,6 +23,9 @@
<version>1.1.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
+ <properties>
+ <sbt.project.name>yarn-stable</sbt.project.name>
+ </properties>
<groupId>org.apache.spark</groupId>
<artifactId>spark-yarn_2.10</artifactId>