From 289373b28cd2546165187de2e6a9185a1257b1e7 Mon Sep 17 00:00:00 2001
From: Josh Rosen <joshrosen@databricks.com>
Date: Sat, 30 Jan 2016 00:20:28 -0800
Subject: [PATCH] [SPARK-6363][BUILD] Make Scala 2.11 the default Scala version
This patch changes Spark's build to make Scala 2.11 the default Scala version. To be clear, this does not mean that Spark will stop supporting Scala 2.10: users will still be able to compile Spark for Scala 2.10 by following the instructions on the "Building Spark" page; however, it does mean that Scala 2.11 will be the default Scala version used by our CI builds (including pull request builds).
The Scala 2.11 compiler is faster than 2.10, so I think we'll be able to look forward to a slight speedup in our CI builds (it looks like it's about 2X faster for the Maven compile-only builds, for instance).
After this patch is merged, I'll update Jenkins to add new compile-only jobs to ensure that Scala 2.10 compilation doesn't break.
Author: Josh Rosen <joshrosen@databricks.com>
Closes #10608 from JoshRosen/SPARK-6363.
---
assembly/pom.xml | 4 +--
common/sketch/pom.xml | 4 +--
core/pom.xml | 4 +--
dev/create-release/release-build.sh | 14 ++++-----
dev/deps/spark-deps-hadoop-2.2 | 31 +++++++++----------
dev/deps/spark-deps-hadoop-2.3 | 31 +++++++++----------
dev/deps/spark-deps-hadoop-2.4 | 31 +++++++++----------
dev/deps/spark-deps-hadoop-2.6 | 31 +++++++++----------
dev/deps/spark-deps-hadoop-2.7 | 31 +++++++++----------
docker-integration-tests/pom.xml | 4 +--
docs/_plugins/copy_api_dirs.rb | 2 +-
docs/building-spark.md | 10 +++---
examples/pom.xml | 4 +--
external/akka/pom.xml | 4 +--
external/flume-assembly/pom.xml | 4 +--
external/flume-sink/pom.xml | 4 +--
external/flume/pom.xml | 4 +--
external/kafka-assembly/pom.xml | 4 +--
external/kafka/pom.xml | 4 +--
external/mqtt-assembly/pom.xml | 4 +--
external/mqtt/pom.xml | 4 +--
external/twitter/pom.xml | 4 +--
external/zeromq/pom.xml | 4 +--
extras/java8-tests/pom.xml | 4 +--
extras/kinesis-asl-assembly/pom.xml | 4 +--
extras/kinesis-asl/pom.xml | 4 +--
extras/spark-ganglia-lgpl/pom.xml | 4 +--
graphx/pom.xml | 4 +--
launcher/pom.xml | 4 +--
mllib/pom.xml | 4 +--
network/common/pom.xml | 4 +--
network/shuffle/pom.xml | 4 +--
network/yarn/pom.xml | 4 +--
pom.xml | 8 ++---
project/MimaBuild.scala | 2 +-
project/MimaExcludes.scala | 6 ++++
project/SparkBuild.scala | 12 +++----
repl/pom.xml | 8 ++---
.../scala/org/apache/spark/repl/Main.scala | 9 +++++-
.../org/apache/spark/repl/ReplSuite.scala | 7 +----
sql/catalyst/pom.xml | 13 ++------
sql/core/pom.xml | 6 ++--
sql/hive-thriftserver/pom.xml | 4 +--
sql/hive/pom.xml | 4 +--
streaming/pom.xml | 4 +--
tags/pom.xml | 4 +--
tools/pom.xml | 4 +--
unsafe/pom.xml | 4 +--
yarn/pom.xml | 4 +--
49 files changed, 186 insertions(+), 194 deletions(-)
diff --git a/assembly/pom.xml b/assembly/pom.xml
index 6c79f91897..477d4931c3 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-assembly_2.10</artifactId>
+ <artifactId>spark-assembly_2.11</artifactId>
<name>Spark Project Assembly</name>
<url>http://spark.apache.org/</url>
<packaging>pom</packaging>
diff --git a/common/sketch/pom.xml b/common/sketch/pom.xml
index 2cafe8c548..442043cb51 100644
--- a/common/sketch/pom.xml
+++ b/common/sketch/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sketch_2.10</artifactId>
+ <artifactId>spark-sketch_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Sketch</name>
<url>http://spark.apache.org/</url>
diff --git a/core/pom.xml b/core/pom.xml
index 0ab170e028..be40d9936a 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_2.10</artifactId>
+ <artifactId>spark-core_2.11</artifactId>
<properties>
<sbt.project.name>core</sbt.project.name>
</properties>
diff --git a/dev/create-release/release-build.sh b/dev/create-release/release-build.sh
index 00bf81120d..2fd7fcc39e 100755
--- a/dev/create-release/release-build.sh
+++ b/dev/create-release/release-build.sh
@@ -134,9 +134,9 @@ if [[ "$1" == "package" ]]; then
cd spark-$SPARK_VERSION-bin-$NAME
- # TODO There should probably be a flag to make-distribution to allow 2.11 support
- if [[ $FLAGS == *scala-2.11* ]]; then
- ./dev/change-scala-version.sh 2.11
+ # TODO There should probably be a flag to make-distribution to allow 2.10 support
+ if [[ $FLAGS == *scala-2.10* ]]; then
+ ./dev/change-scala-version.sh 2.10
fi
export ZINC_PORT=$ZINC_PORT
@@ -228,8 +228,8 @@ if [[ "$1" == "publish-snapshot" ]]; then
$MVN -DzincPort=$ZINC_PORT --settings $tmp_settings -DskipTests $PUBLISH_PROFILES \
-Phive-thriftserver deploy
- ./dev/change-scala-version.sh 2.11
- $MVN -DzincPort=$ZINC_PORT -Dscala-2.11 --settings $tmp_settings \
+ ./dev/change-scala-version.sh 2.10
+ $MVN -DzincPort=$ZINC_PORT -Dscala-2.10 --settings $tmp_settings \
-DskipTests $PUBLISH_PROFILES clean deploy
# Clean-up Zinc nailgun process
@@ -266,9 +266,9 @@ if [[ "$1" == "publish-release" ]]; then
$MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -DskipTests $PUBLISH_PROFILES \
-Phive-thriftserver clean install
- ./dev/change-scala-version.sh 2.11
+ ./dev/change-scala-version.sh 2.10
- $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.11 \
+ $MVN -DzincPort=$ZINC_PORT -Dmaven.repo.local=$tmp_repo -Dscala-2.10 \
-DskipTests $PUBLISH_PROFILES clean install
# Clean-up Zinc nailgun process
diff --git a/dev/deps/spark-deps-hadoop-2.2 b/dev/deps/spark-deps-hadoop-2.2
index 4d9937c5cb..3a14499d9b 100644
--- a/dev/deps/spark-deps-hadoop-2.2
+++ b/dev/deps/spark-deps-hadoop-2.2
@@ -14,13 +14,13 @@ avro-ipc-1.7.7-tests.jar
avro-ipc-1.7.7.jar
avro-mapred-1.7.7-hadoop2.jar
bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
calcite-avatica-1.2.0-incubating.jar
calcite-core-1.2.0-incubating.jar
calcite-linq4j-1.2.0-incubating.jar
chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
commons-beanutils-1.7.0.jar
commons-beanutils-core-1.8.0.jar
commons-cli-1.2.jar
@@ -86,10 +86,9 @@ jackson-core-asl-1.9.13.jar
jackson-databind-2.5.3.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
jackson-xc-1.9.13.jar
janino-2.7.8.jar
-jansi-1.4.jar
javax.inject-1.jar
javax.servlet-3.0.0.v201112011016.jar
javax.servlet-3.1.jar
@@ -111,15 +110,14 @@ jets3t-0.7.1.jar
jettison-1.1.jar
jetty-all-7.6.0.v20120127.jar
jetty-util-6.1.26.jar
-jline-2.10.5.jar
jline-2.12.jar
joda-time-2.9.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
jsr305-1.3.9.jar
jta-1.1.jar
jtransforms-2.4.0.jar
@@ -158,19 +156,20 @@ pmml-schema-1.2.7.jar
protobuf-java-2.5.0.jar
py4j-0.9.1.jar
pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
servlet-api-2.5.jar
slf4j-api-1.7.10.jar
slf4j-log4j12-1.7.10.jar
snappy-0.2.jar
snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
stax-api-1.0-2.jar
stax-api-1.0.1.jar
stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.3 b/dev/deps/spark-deps-hadoop-2.3
index fd659ee20d..615836b3d3 100644
--- a/dev/deps/spark-deps-hadoop-2.3
+++ b/dev/deps/spark-deps-hadoop-2.3
@@ -16,13 +16,13 @@ avro-mapred-1.7.7-hadoop2.jar
base64-2.3.8.jar
bcprov-jdk15on-1.51.jar
bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
calcite-avatica-1.2.0-incubating.jar
calcite-core-1.2.0-incubating.jar
calcite-linq4j-1.2.0-incubating.jar
chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
commons-beanutils-1.7.0.jar
commons-beanutils-core-1.8.0.jar
commons-cli-1.2.jar
@@ -81,10 +81,9 @@ jackson-core-asl-1.9.13.jar
jackson-databind-2.5.3.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
jackson-xc-1.9.13.jar
janino-2.7.8.jar
-jansi-1.4.jar
java-xmlbuilder-1.0.jar
javax.inject-1.jar
javax.servlet-3.0.0.v201112011016.jar
@@ -102,15 +101,14 @@ jettison-1.1.jar
jetty-6.1.26.jar
jetty-all-7.6.0.v20120127.jar
jetty-util-6.1.26.jar
-jline-2.10.5.jar
jline-2.12.jar
joda-time-2.9.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
jsr305-1.3.9.jar
jta-1.1.jar
jtransforms-2.4.0.jar
@@ -149,19 +147,20 @@ pmml-schema-1.2.7.jar
protobuf-java-2.5.0.jar
py4j-0.9.1.jar
pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
servlet-api-2.5.jar
slf4j-api-1.7.10.jar
slf4j-log4j12-1.7.10.jar
snappy-0.2.jar
snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
stax-api-1.0-2.jar
stax-api-1.0.1.jar
stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.4 b/dev/deps/spark-deps-hadoop-2.4
index afae3deb9a..f275226f1d 100644
--- a/dev/deps/spark-deps-hadoop-2.4
+++ b/dev/deps/spark-deps-hadoop-2.4
@@ -16,13 +16,13 @@ avro-mapred-1.7.7-hadoop2.jar
base64-2.3.8.jar
bcprov-jdk15on-1.51.jar
bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
calcite-avatica-1.2.0-incubating.jar
calcite-core-1.2.0-incubating.jar
calcite-linq4j-1.2.0-incubating.jar
chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
commons-beanutils-1.7.0.jar
commons-beanutils-core-1.8.0.jar
commons-cli-1.2.jar
@@ -81,10 +81,9 @@ jackson-core-asl-1.9.13.jar
jackson-databind-2.5.3.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
jackson-xc-1.9.13.jar
janino-2.7.8.jar
-jansi-1.4.jar
java-xmlbuilder-1.0.jar
javax.inject-1.jar
javax.servlet-3.0.0.v201112011016.jar
@@ -103,15 +102,14 @@ jettison-1.1.jar
jetty-6.1.26.jar
jetty-all-7.6.0.v20120127.jar
jetty-util-6.1.26.jar
-jline-2.10.5.jar
jline-2.12.jar
joda-time-2.9.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
jsr305-1.3.9.jar
jta-1.1.jar
jtransforms-2.4.0.jar
@@ -150,19 +148,20 @@ pmml-schema-1.2.7.jar
protobuf-java-2.5.0.jar
py4j-0.9.1.jar
pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
servlet-api-2.5.jar
slf4j-api-1.7.10.jar
slf4j-log4j12-1.7.10.jar
snappy-0.2.jar
snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
stax-api-1.0-2.jar
stax-api-1.0.1.jar
stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index 5a6460136a..21432a16e3 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -20,13 +20,13 @@ avro-mapred-1.7.7-hadoop2.jar
base64-2.3.8.jar
bcprov-jdk15on-1.51.jar
bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
calcite-avatica-1.2.0-incubating.jar
calcite-core-1.2.0-incubating.jar
calcite-linq4j-1.2.0-incubating.jar
chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
commons-beanutils-1.7.0.jar
commons-beanutils-core-1.8.0.jar
commons-cli-1.2.jar
@@ -87,10 +87,9 @@ jackson-core-asl-1.9.13.jar
jackson-databind-2.5.3.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
jackson-xc-1.9.13.jar
janino-2.7.8.jar
-jansi-1.4.jar
java-xmlbuilder-1.0.jar
javax.inject-1.jar
javax.servlet-3.0.0.v201112011016.jar
@@ -109,15 +108,14 @@ jettison-1.1.jar
jetty-6.1.26.jar
jetty-all-7.6.0.v20120127.jar
jetty-util-6.1.26.jar
-jline-2.10.5.jar
jline-2.12.jar
joda-time-2.9.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
jsr305-1.3.9.jar
jta-1.1.jar
jtransforms-2.4.0.jar
@@ -156,19 +154,20 @@ pmml-schema-1.2.7.jar
protobuf-java-2.5.0.jar
py4j-0.9.1.jar
pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
servlet-api-2.5.jar
slf4j-api-1.7.10.jar
slf4j-log4j12-1.7.10.jar
snappy-0.2.jar
snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
stax-api-1.0-2.jar
stax-api-1.0.1.jar
stream-2.7.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 70083e7f3d..20e09cd002 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -20,13 +20,13 @@ avro-mapred-1.7.7-hadoop2.jar
base64-2.3.8.jar
bcprov-jdk15on-1.51.jar
bonecp-0.8.0.RELEASE.jar
-breeze-macros_2.10-0.11.2.jar
-breeze_2.10-0.11.2.jar
+breeze-macros_2.11-0.11.2.jar
+breeze_2.11-0.11.2.jar
calcite-avatica-1.2.0-incubating.jar
calcite-core-1.2.0-incubating.jar
calcite-linq4j-1.2.0-incubating.jar
chill-java-0.5.0.jar
-chill_2.10-0.5.0.jar
+chill_2.11-0.5.0.jar
commons-beanutils-1.7.0.jar
commons-beanutils-core-1.8.0.jar
commons-cli-1.2.jar
@@ -87,10 +87,9 @@ jackson-core-asl-1.9.13.jar
jackson-databind-2.5.3.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-scala_2.10-2.5.3.jar
+jackson-module-scala_2.11-2.5.3.jar
jackson-xc-1.9.13.jar
janino-2.7.8.jar
-jansi-1.4.jar
java-xmlbuilder-1.0.jar
javax.inject-1.jar
javax.servlet-3.0.0.v201112011016.jar
@@ -109,15 +108,14 @@ jettison-1.1.jar
jetty-6.1.26.jar
jetty-all-7.6.0.v20120127.jar
jetty-util-6.1.26.jar
-jline-2.10.5.jar
jline-2.12.jar
joda-time-2.9.jar
jodd-core-3.5.2.jar
jpam-1.1.jar
json-20090211.jar
-json4s-ast_2.10-3.2.10.jar
-json4s-core_2.10-3.2.10.jar
-json4s-jackson_2.10-3.2.10.jar
+json4s-ast_2.11-3.2.10.jar
+json4s-core_2.11-3.2.10.jar
+json4s-jackson_2.11-3.2.10.jar
jsp-api-2.1.jar
jsr305-1.3.9.jar
jta-1.1.jar
@@ -157,19 +155,20 @@ pmml-schema-1.2.7.jar
protobuf-java-2.5.0.jar
py4j-0.9.1.jar
pyrolite-4.9.jar
-quasiquotes_2.10-2.0.0-M8.jar
reflectasm-1.07-shaded.jar
-scala-compiler-2.10.5.jar
-scala-library-2.10.5.jar
-scala-reflect-2.10.5.jar
-scalap-2.10.5.jar
+scala-compiler-2.11.7.jar
+scala-library-2.11.7.jar
+scala-parser-combinators_2.11-1.0.4.jar
+scala-reflect-2.11.7.jar
+scala-xml_2.11-1.0.2.jar
+scalap-2.11.7.jar
servlet-api-2.5.jar
slf4j-api-1.7.10.jar
slf4j-log4j12-1.7.10.jar
snappy-0.2.jar
snappy-java-1.1.2.jar
-spire-macros_2.10-0.7.4.jar
-spire_2.10-0.7.4.jar
+spire-macros_2.11-0.7.4.jar
+spire_2.11-0.7.4.jar
stax-api-1.0-2.jar
stax-api-1.0.1.jar
stream-2.7.0.jar
diff --git a/docker-integration-tests/pom.xml b/docker-integration-tests/pom.xml
index 78b638ecfa..833ca29cd8 100644
--- a/docker-integration-tests/pom.xml
+++ b/docker-integration-tests/pom.xml
@@ -21,12 +21,12 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
- <artifactId>spark-docker-integration-tests_2.10</artifactId>
+ <artifactId>spark-docker-integration-tests_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Docker Integration Tests</name>
<url>http://spark.apache.org/</url>
diff --git a/docs/_plugins/copy_api_dirs.rb b/docs/_plugins/copy_api_dirs.rb
index 174c202e37..f926d67e6b 100644
--- a/docs/_plugins/copy_api_dirs.rb
+++ b/docs/_plugins/copy_api_dirs.rb
@@ -37,7 +37,7 @@ if not (ENV['SKIP_API'] == '1')
# Copy over the unified ScalaDoc for all projects to api/scala.
# This directory will be copied over to _site when `jekyll` command is run.
- source = "../target/scala-2.10/unidoc"
+ source = "../target/scala-2.11/unidoc"
dest = "api/scala"
puts "Making directory " + dest
diff --git a/docs/building-spark.md b/docs/building-spark.md
index e1abcf1be5..975e1b295c 100644
--- a/docs/building-spark.md
+++ b/docs/building-spark.md
@@ -114,13 +114,11 @@ By default Spark will build with Hive 0.13.1 bindings.
mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -Phive -Phive-thriftserver -DskipTests clean package
{% endhighlight %}
-# Building for Scala 2.11
-To produce a Spark package compiled with Scala 2.11, use the `-Dscala-2.11` property:
+# Building for Scala 2.10
+To produce a Spark package compiled with Scala 2.10, use the `-Dscala-2.10` property:
- ./dev/change-scala-version.sh 2.11
- mvn -Pyarn -Phadoop-2.4 -Dscala-2.11 -DskipTests clean package
-
-Spark does not yet support its JDBC component for Scala 2.11.
+ ./dev/change-scala-version.sh 2.10
+ mvn -Pyarn -Phadoop-2.4 -Dscala-2.10 -DskipTests clean package
# Spark Tests in Maven
diff --git a/examples/pom.xml b/examples/pom.xml
index 9437cee2ab..82baa9085b 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-examples_2.10</artifactId>
+ <artifactId>spark-examples_2.11</artifactId>
<properties>
<sbt.project.name>examples</sbt.project.name>
</properties>
diff --git a/external/akka/pom.xml b/external/akka/pom.xml
index 06c8e8aaab..bbe644e3b3 100644
--- a/external/akka/pom.xml
+++ b/external/akka/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-akka_2.10</artifactId>
+ <artifactId>spark-streaming-akka_2.11</artifactId>
<properties>
<sbt.project.name>streaming-akka</sbt.project.name>
</properties>
diff --git a/external/flume-assembly/pom.xml b/external/flume-assembly/pom.xml
index b2c377fe4c..ac15b93c04 100644
--- a/external/flume-assembly/pom.xml
+++ b/external/flume-assembly/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-flume-assembly_2.10</artifactId>
+ <artifactId>spark-streaming-flume-assembly_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project External Flume Assembly</name>
<url>http://spark.apache.org/</url>
diff --git a/external/flume-sink/pom.xml b/external/flume-sink/pom.xml
index 4b6485ee0a..e4effe158c 100644
--- a/external/flume-sink/pom.xml
+++ b/external/flume-sink/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-flume-sink_2.10</artifactId>
+ <artifactId>spark-streaming-flume-sink_2.11</artifactId>
<properties>
<sbt.project.name>streaming-flume-sink</sbt.project.name>
</properties>
diff --git a/external/flume/pom.xml b/external/flume/pom.xml
index a79656c6f7..d650dd034d 100644
--- a/external/flume/pom.xml
+++ b/external/flume/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-flume_2.10</artifactId>
+ <artifactId>spark-streaming-flume_2.11</artifactId>
<properties>
<sbt.project.name>streaming-flume</sbt.project.name>
</properties>
diff --git a/external/kafka-assembly/pom.xml b/external/kafka-assembly/pom.xml
index 0c466b3c4a..62818f5e8f 100644
--- a/external/kafka-assembly/pom.xml
+++ b/external/kafka-assembly/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kafka-assembly_2.10</artifactId>
+ <artifactId>spark-streaming-kafka-assembly_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project External Kafka Assembly</name>
<url>http://spark.apache.org/</url>
diff --git a/external/kafka/pom.xml b/external/kafka/pom.xml
index 5180ab6dba..68d52e9339 100644
--- a/external/kafka/pom.xml
+++ b/external/kafka/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kafka_2.10</artifactId>
+ <artifactId>spark-streaming-kafka_2.11</artifactId>
<properties>
<sbt.project.name>streaming-kafka</sbt.project.name>
</properties>
diff --git a/external/mqtt-assembly/pom.xml b/external/mqtt-assembly/pom.xml
index c4a1ae26ea..ac2a3f65ed 100644
--- a/external/mqtt-assembly/pom.xml
+++ b/external/mqtt-assembly/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-mqtt-assembly_2.10</artifactId>
+ <artifactId>spark-streaming-mqtt-assembly_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project External MQTT Assembly</name>
<url>http://spark.apache.org/</url>
diff --git a/external/mqtt/pom.xml b/external/mqtt/pom.xml
index d3a2bf5825..d0d968782c 100644
--- a/external/mqtt/pom.xml
+++ b/external/mqtt/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-mqtt_2.10</artifactId>
+ <artifactId>spark-streaming-mqtt_2.11</artifactId>
<properties>
<sbt.project.name>streaming-mqtt</sbt.project.name>
</properties>
diff --git a/external/twitter/pom.xml b/external/twitter/pom.xml
index 7b628b09ea..5d4053afcb 100644
--- a/external/twitter/pom.xml
+++ b/external/twitter/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-twitter_2.10</artifactId>
+ <artifactId>spark-streaming-twitter_2.11</artifactId>
<properties>
<sbt.project.name>streaming-twitter</sbt.project.name>
</properties>
diff --git a/external/zeromq/pom.xml b/external/zeromq/pom.xml
index 7781aaeed9..f16bc0f319 100644
--- a/external/zeromq/pom.xml
+++ b/external/zeromq/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-zeromq_2.10</artifactId>
+ <artifactId>spark-streaming-zeromq_2.11</artifactId>
<properties>
<sbt.project.name>streaming-zeromq</sbt.project.name>
</properties>
diff --git a/extras/java8-tests/pom.xml b/extras/java8-tests/pom.xml
index 4dfe3b654d..0ad9c5303a 100644
--- a/extras/java8-tests/pom.xml
+++ b/extras/java8-tests/pom.xml
@@ -19,13 +19,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>java8-tests_2.10</artifactId>
+ <artifactId>java8-tests_2.11</artifactId>
<packaging>pom</packaging>
<name>Spark Project Java8 Tests POM</name>
diff --git a/extras/kinesis-asl-assembly/pom.xml b/extras/kinesis-asl-assembly/pom.xml
index 601080c2e6..d1c38c7ca5 100644
--- a/extras/kinesis-asl-assembly/pom.xml
+++ b/extras/kinesis-asl-assembly/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kinesis-asl-assembly_2.10</artifactId>
+ <artifactId>spark-streaming-kinesis-asl-assembly_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Kinesis Assembly</name>
<url>http://spark.apache.org/</url>
diff --git a/extras/kinesis-asl/pom.xml b/extras/kinesis-asl/pom.xml
index 20e2c5e0ff..935155eb5d 100644
--- a/extras/kinesis-asl/pom.xml
+++ b/extras/kinesis-asl/pom.xml
@@ -19,14 +19,14 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<!-- Kinesis integration is not included by default due to ASL-licensed code. -->
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming-kinesis-asl_2.10</artifactId>
+ <artifactId>spark-streaming-kinesis-asl_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Kinesis Integration</name>
diff --git a/extras/spark-ganglia-lgpl/pom.xml b/extras/spark-ganglia-lgpl/pom.xml
index b046a10a04..bfb92791de 100644
--- a/extras/spark-ganglia-lgpl/pom.xml
+++ b/extras/spark-ganglia-lgpl/pom.xml
@@ -19,14 +19,14 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<!-- Ganglia integration is not included by default due to LGPL-licensed code -->
<groupId>org.apache.spark</groupId>
- <artifactId>spark-ganglia-lgpl_2.10</artifactId>
+ <artifactId>spark-ganglia-lgpl_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Ganglia Integration</name>
diff --git a/graphx/pom.xml b/graphx/pom.xml
index 388a0ef06a..1813f383cd 100644
--- a/graphx/pom.xml
+++ b/graphx/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-graphx_2.10</artifactId>
+ <artifactId>spark-graphx_2.11</artifactId>
<properties>
<sbt.project.name>graphx</sbt.project.name>
</properties>
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 135866cea2..ef73194882 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-launcher_2.10</artifactId>
+ <artifactId>spark-launcher_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Launcher</name>
<url>http://spark.apache.org/</url>
diff --git a/mllib/pom.xml b/mllib/pom.xml
index 42af2b8b3e..816f3f6830 100644
--- a/mllib/pom.xml
+++ b/mllib/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-mllib_2.10</artifactId>
+ <artifactId>spark-mllib_2.11</artifactId>
<properties>
<sbt.project.name>mllib</sbt.project.name>
</properties>
diff --git a/network/common/pom.xml b/network/common/pom.xml
index eda2b73070..bd507c2cb6 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-network-common_2.10</artifactId>
+ <artifactId>spark-network-common_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Networking</name>
<url>http://spark.apache.org/</url>
diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml
index f9aa7e2dd1..810ec10ca0 100644
--- a/network/shuffle/pom.xml
+++ b/network/shuffle/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-network-shuffle_2.10</artifactId>
+ <artifactId>spark-network-shuffle_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Shuffle Streaming Service</name>
<url>http://spark.apache.org/</url>
diff --git a/network/yarn/pom.xml b/network/yarn/pom.xml
index a19cbb04b1..a28785b16e 100644
--- a/network/yarn/pom.xml
+++ b/network/yarn/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-network-yarn_2.10</artifactId>
+ <artifactId>spark-network-yarn_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project YARN Shuffle Service</name>
<url>http://spark.apache.org/</url>
diff --git a/pom.xml b/pom.xml
index fb7750602c..d0387aca66 100644
--- a/pom.xml
+++ b/pom.xml
@@ -25,7 +25,7 @@
<version>14</version>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<packaging>pom</packaging>
<name>Spark Project Parent POM</name>
@@ -165,7 +165,7 @@
<!-- managed up from 3.2.1 for SPARK-11652 -->
<commons.collections.version>3.2.2</commons.collections.version>
<scala.version>2.10.5</scala.version>
- <scala.binary.version>2.10</scala.binary.version>
+ <scala.binary.version>2.11</scala.binary.version>
<jline.version>${scala.version}</jline.version>
<jline.groupid>org.scala-lang</jline.groupid>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
@@ -2456,7 +2456,7 @@
<profile>
<id>scala-2.10</id>
<activation>
- <property><name>!scala-2.11</name></property>
+ <property><name>scala-2.10</name></property>
</activation>
<properties>
<scala.version>2.10.5</scala.version>
@@ -2488,7 +2488,7 @@
<profile>
<id>scala-2.11</id>
<activation>
- <property><name>scala-2.11</name></property>
+ <property><name>!scala-2.10</name></property>
</activation>
<properties>
<scala.version>2.11.7</scala.version>
diff --git a/project/MimaBuild.scala b/project/MimaBuild.scala
index 41856443af..4adf64a5a0 100644
--- a/project/MimaBuild.scala
+++ b/project/MimaBuild.scala
@@ -95,7 +95,7 @@ object MimaBuild {
// because spark-streaming-mqtt(1.6.0) depends on it.
// Remove the setting on updating previousSparkVersion.
val previousSparkVersion = "1.6.0"
- val fullId = "spark-" + projectRef.project + "_2.10"
+ val fullId = "spark-" + projectRef.project + "_2.11"
mimaDefaultSettings ++
Seq(previousArtifact := Some(organization % fullId % previousSparkVersion),
binaryIssueFilters ++= ignoredABIProblems(sparkHome, version.value),
diff --git a/project/MimaExcludes.scala b/project/MimaExcludes.scala
index a3ae4d2b73..3748e07f88 100644
--- a/project/MimaExcludes.scala
+++ b/project/MimaExcludes.scala
@@ -220,6 +220,12 @@ object MimaExcludes {
// SPARK-11622 Make LibSVMRelation extends HadoopFsRelation and Add LibSVMOutputWriter
ProblemFilters.exclude[MissingTypesProblem]("org.apache.spark.ml.source.libsvm.DefaultSource"),
ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.ml.source.libsvm.DefaultSource.createRelation")
+ ) ++ Seq(
+ // SPARK-6363 Make Scala 2.11 the default Scala version
+ ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.SparkContext.cleanup"),
+ ProblemFilters.exclude[MissingMethodProblem]("org.apache.spark.SparkContext.metadataCleaner"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnDriverEndpoint"),
+ ProblemFilters.exclude[MissingClassProblem]("org.apache.spark.scheduler.cluster.YarnSchedulerBackend$YarnSchedulerEndpoint")
)
case v if v.startsWith("1.6") =>
Seq(
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 4224a65a82..550b5bad8a 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -119,11 +119,11 @@ object SparkBuild extends PomBuild {
v.split("(\\s+|,)").filterNot(_.isEmpty).map(_.trim.replaceAll("-P", "")).toSeq
}
- if (System.getProperty("scala-2.11") == "") {
- // To activate scala-2.11 profile, replace empty property value to non-empty value
+ if (System.getProperty("scala-2.10") == "") {
+ // To activate scala-2.10 profile, replace empty property value to non-empty value
// in the same way as Maven which handles -Dname as -Dname=true before executes build process.
// see: https://github.com/apache/maven/blob/maven-3.0.4/maven-embedder/src/main/java/org/apache/maven/cli/MavenCli.java#L1082
- System.setProperty("scala-2.11", "true")
+ System.setProperty("scala-2.10", "true")
}
profiles
}
@@ -382,7 +382,7 @@ object OldDeps {
lazy val project = Project("oldDeps", file("dev"), settings = oldDepsSettings)
def versionArtifact(id: String): Option[sbt.ModuleID] = {
- val fullId = id + "_2.10"
+ val fullId = id + "_2.11"
Some("org.apache.spark" % fullId % "1.2.0")
}
@@ -390,7 +390,7 @@ object OldDeps {
name := "old-deps",
scalaVersion := "2.10.5",
libraryDependencies := Seq("spark-streaming-mqtt", "spark-streaming-zeromq",
- "spark-streaming-flume", "spark-streaming-kafka", "spark-streaming-twitter",
+ "spark-streaming-flume", "spark-streaming-twitter",
"spark-streaming", "spark-mllib", "spark-graphx",
"spark-core").map(versionArtifact(_).get intransitive())
)
@@ -704,7 +704,7 @@ object Java8TestSettings {
lazy val settings = Seq(
javacJVMVersion := "1.8",
// Targeting Java 8 bytecode is only supported in Scala 2.11.4 and higher:
- scalacJVMVersion := (if (System.getProperty("scala-2.11") == "true") "1.8" else "1.7")
+ scalacJVMVersion := (if (System.getProperty("scala-2.10") == "true") "1.7" else "1.8")
)
}
diff --git a/repl/pom.xml b/repl/pom.xml
index efc3dd452e..0f396c9b80 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-repl_2.10</artifactId>
+ <artifactId>spark-repl_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project REPL</name>
<url>http://spark.apache.org/</url>
@@ -159,7 +159,7 @@
<profile>
<id>scala-2.10</id>
<activation>
- <property><name>!scala-2.11</name></property>
+ <property><name>scala-2.10</name></property>
</activation>
<dependencies>
<dependency>
@@ -173,7 +173,7 @@
<profile>
<id>scala-2.11</id>
<activation>
- <property><name>scala-2.11</name></property>
+ <property><name>!scala-2.10</name></property>
</activation>
<properties>
<extra.source.dir>scala-2.11/src/main/scala</extra.source.dir>
diff --git a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
index bb3081d129..07ba28bb07 100644
--- a/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
+++ b/repl/scala-2.11/src/main/scala/org/apache/spark/repl/Main.scala
@@ -33,7 +33,8 @@ object Main extends Logging {
var sparkContext: SparkContext = _
var sqlContext: SQLContext = _
- var interp = new SparkILoop // this is a public var because tests reset it.
+ // this is a public var because tests reset it.
+ var interp: SparkILoop = _
private var hasErrors = false
@@ -43,6 +44,12 @@ object Main extends Logging {
}
def main(args: Array[String]) {
+ doMain(args, new SparkILoop)
+ }
+
+ // Visible for testing
+ private[repl] def doMain(args: Array[String], _interp: SparkILoop): Unit = {
+ interp = _interp
val interpArguments = List(
"-Yrepl-class-based",
"-Yrepl-outdir", s"${outputDir.getAbsolutePath}",
diff --git a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
index 63f3688c9e..b9ed79da42 100644
--- a/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
+++ b/repl/scala-2.11/src/test/scala/org/apache/spark/repl/ReplSuite.scala
@@ -50,12 +50,7 @@ class ReplSuite extends SparkFunSuite {
System.setProperty(CONF_EXECUTOR_CLASSPATH, classpath)
System.setProperty("spark.master", master)
- val interp = {
- new SparkILoop(in, new PrintWriter(out))
- }
- org.apache.spark.repl.Main.interp = interp
- Main.main(Array("-classpath", classpath)) // call main
- org.apache.spark.repl.Main.interp = null
+ Main.doMain(Array("-classpath", classpath), new SparkILoop(in, new PrintWriter(out)))
if (oldExecutorClasspath != null) {
System.setProperty(CONF_EXECUTOR_CLASSPATH, oldExecutorClasspath)
diff --git a/sql/catalyst/pom.xml b/sql/catalyst/pom.xml
index 76ca3f3bb1..c2ad9b99f3 100644
--- a/sql/catalyst/pom.xml
+++ b/sql/catalyst/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_2.10</artifactId>
+ <artifactId>spark-catalyst_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Catalyst</name>
<url>http://spark.apache.org/</url>
@@ -127,13 +127,4 @@
</plugin>
</plugins>
</build>
- <profiles>
- <!-- Quasiquotes are merged into scala reflect from scala 2.11 onwards. -->
- <profile>
- <id>scala-2.10</id>
- <activation>
- <property><name>!scala-2.11</name></property>
- </activation>
- </profile>
- </profiles>
</project>
diff --git a/sql/core/pom.xml b/sql/core/pom.xml
index 4bb55f6b7f..89e01fc015 100644
--- a/sql/core/pom.xml
+++ b/sql/core/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.10</artifactId>
+ <artifactId>spark-sql_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project SQL</name>
<url>http://spark.apache.org/</url>
@@ -44,7 +44,7 @@
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sketch_2.10</artifactId>
+ <artifactId>spark-sketch_2.11</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
diff --git a/sql/hive-thriftserver/pom.xml b/sql/hive-thriftserver/pom.xml
index 435e565f63..c8d17bd468 100644
--- a/sql/hive-thriftserver/pom.xml
+++ b/sql/hive-thriftserver/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive-thriftserver_2.10</artifactId>
+ <artifactId>spark-hive-thriftserver_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Hive Thrift Server</name>
<url>http://spark.apache.org/</url>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index cd0c2aeb93..14cf9acf09 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.10</artifactId>
+ <artifactId>spark-hive_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Hive</name>
<url>http://spark.apache.org/</url>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 39cbd0d00f..7d409c5d3b 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -20,13 +20,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-streaming_2.10</artifactId>
+ <artifactId>spark-streaming_2.11</artifactId>
<properties>
<sbt.project.name>streaming</sbt.project.name>
</properties>
diff --git a/tags/pom.xml b/tags/pom.xml
index 9e4610dae7..3e8e6f6182 100644
--- a/tags/pom.xml
+++ b/tags/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-test-tags_2.10</artifactId>
+ <artifactId>spark-test-tags_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Test Tags</name>
<url>http://spark.apache.org/</url>
diff --git a/tools/pom.xml b/tools/pom.xml
index 30cbb6a5a5..b3a5ae2771 100644
--- a/tools/pom.xml
+++ b/tools/pom.xml
@@ -19,13 +19,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-tools_2.10</artifactId>
+ <artifactId>spark-tools_2.11</artifactId>
<properties>
<sbt.project.name>tools</sbt.project.name>
</properties>
diff --git a/unsafe/pom.xml b/unsafe/pom.xml
index 21fef3415a..75fea556ee 100644
--- a/unsafe/pom.xml
+++ b/unsafe/pom.xml
@@ -21,13 +21,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-unsafe_2.10</artifactId>
+ <artifactId>spark-unsafe_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project Unsafe</name>
<url>http://spark.apache.org/</url>
diff --git a/yarn/pom.xml b/yarn/pom.xml
index a8c122fd40..328bb6678d 100644
--- a/yarn/pom.xml
+++ b/yarn/pom.xml
@@ -19,13 +19,13 @@
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-parent_2.10</artifactId>
+ <artifactId>spark-parent_2.11</artifactId>
<version>2.0.0-SNAPSHOT</version>
<relativePath>../pom.xml</relativePath>
</parent>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-yarn_2.10</artifactId>
+ <artifactId>spark-yarn_2.11</artifactId>
<packaging>jar</packaging>
<name>Spark Project YARN</name>
<properties>
--
GitLab