diff --git a/core/pom.xml b/core/pom.xml
index 36c71e67b56aa7a5e656830fb44442f954afa3fc..c24c7be204087d1daa4eb28a153e7a8918d8a528 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -38,12 +38,6 @@
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
- <exclusions>
- <exclusion>
- <groupId>commons-logging</groupId>
- <artifactId>commons-logging</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md
index e447dfea3bac9715d4502bc94fd1441040ddf410..cac01ded60d94b9b781de39e09e1adf72fbbafac 100644
--- a/docs/building-with-maven.md
+++ b/docs/building-with-maven.md
@@ -29,9 +29,22 @@ You can fix this by setting the `MAVEN_OPTS` variable as discussed before.
## Specifying the Hadoop version ##
-Because HDFS is not protocol-compatible across versions, if you want to read from HDFS, you'll need to build Spark against the specific HDFS version in your environment. You can do this through the "hadoop.version" property. If unset, Spark will build against Hadoop 1.0.4 by default.
-
-For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions without YARN, use:
+Because HDFS is not protocol-compatible across versions, if you want to read from HDFS, you'll need to build Spark against the specific HDFS version in your environment. You can do this through the "hadoop.version" property. If unset, Spark will build against Hadoop 1.0.4 by default. Note that certain build profiles are required for particular Hadoop versions:
+
+<table class="table">
+ <thead>
+ <tr><th>Hadoop version</th><th>Profile required</th></tr>
+ </thead>
+ <tbody>
+ <tr><td>0.23.x</td><td>hadoop-0.23</td></tr>
+ <tr><td>1.x to 2.1.x</td><td>(none)</td></tr>
+ <tr><td>2.2.x</td><td>hadoop-2.2</td></tr>
+ <tr><td>2.3.x</td><td>hadoop-2.3</td></tr>
+ <tr><td>2.4.x</td><td>hadoop-2.4</td></tr>
+ </tbody>
+</table>
+
+For Apache Hadoop versions 1.x, Cloudera CDH "mr1" distributions, and other Hadoop versions without YARN, use:
# Apache Hadoop 1.2.1
$ mvn -Dhadoop.version=1.2.1 -DskipTests clean package
@@ -42,22 +55,40 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit
# Apache Hadoop 0.23.x
$ mvn -Phadoop-0.23 -Dhadoop.version=0.23.7 -DskipTests clean package
-For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property. Note that Hadoop 0.23.X requires a special `-Phadoop-0.23` profile:
+For Apache Hadoop 2.x, 0.23.x, Cloudera CDH, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and optionally set the "yarn.version" property if it is different from "hadoop.version". The additional build profile required depends on the YARN version:
+
+<table class="table">
+ <thead>
+ <tr><th>YARN version</th><th>Profile required</th></tr>
+ </thead>
+ <tbody>
+ <tr><td>0.23.x to 2.1.x</td><td>yarn-alpha</td></tr>
+ <tr><td>2.2.x and later</td><td>yarn</td></tr>
+ </tbody>
+</table>
+
+Examples:
# Apache Hadoop 2.0.5-alpha
$ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -DskipTests clean package
- # Cloudera CDH 4.2.0 with MapReduce v2
+ # Cloudera CDH 4.2.0
$ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -DskipTests clean package
- # Apache Hadoop 2.2.X (e.g. 2.2.0 as below) and newer
- $ mvn -Pyarn -Dhadoop.version=2.2.0 -DskipTests clean package
-
# Apache Hadoop 0.23.x
- $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package
+ $ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -DskipTests clean package
+
+ # Apache Hadoop 2.2.X
+ $ mvn -Pyarn -Phadoop-2.2 -Dhadoop.version=2.2.0 -DskipTests clean package
+
+ # Apache Hadoop 2.3.X
+ $ mvn -Pyarn -Phadoop-2.3 -Dhadoop.version=2.3.0 -DskipTests clean package
+
+ # Apache Hadoop 2.4.X
+ $ mvn -Pyarn -Phadoop-2.4 -Dhadoop.version=2.4.0 -DskipTests clean package
# Different versions of HDFS and YARN.
- $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package
+ $ mvn -Pyarn-alpha -Phadoop-2.3 -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package
## Spark Tests in Maven ##
diff --git a/pom.xml b/pom.xml
index 08c3ac64436b91c4dd986ca53a903c87426dc9fd..e4b5c36d6917685c9d630bd806069733a2a4e997 100644
--- a/pom.xml
+++ b/pom.xml
@@ -129,6 +129,7 @@
<chill.version>0.3.6</chill.version>
<codahale.metrics.version>3.0.0</codahale.metrics.version>
<avro.version>1.7.4</avro.version>
+ <jets3t.version>0.7.1</jets3t.version>
<PermGen>64m</PermGen>
<MaxPermGen>512m</MaxPermGen>
@@ -560,10 +561,18 @@
</exclusion>
</exclusions>
</dependency>
+ <!-- See SPARK-1556 for info on this dependency: -->
<dependency>
<groupId>net.java.dev.jets3t</groupId>
<artifactId>jets3t</artifactId>
- <version>0.7.1</version>
+ <version>${jets3t.version}</version>
+ <scope>runtime</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>commons-logging</groupId>
+ <artifactId>commons-logging</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
@@ -843,36 +852,6 @@
</build>
<profiles>
- <!-- SPARK-1121: Adds an explicit dependency on Avro to work around a Hadoop 0.23.X issue -->
- <profile>
- <id>hadoop-0.23</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
- </dependencies>
- </profile>
-
- <profile>
- <id>yarn-alpha</id>
- <properties>
- <hadoop.major.version>2</hadoop.major.version>
- <!-- 0.23.* is same as 2.0.* - except hardened to run production jobs -->
- <hadoop.version>0.23.7</hadoop.version>
- <!--<hadoop.version>2.0.5-alpha</hadoop.version> -->
- </properties>
- <dependencies>
- <dependency>
- <groupId>org.apache.avro</groupId>
- <artifactId>avro</artifactId>
- </dependency>
- </dependencies>
- <modules>
- <module>yarn</module>
- </modules>
-
- </profile>
<!-- Ganglia integration is not included by default due to LGPL-licensed code -->
<profile>
@@ -907,17 +886,54 @@
</profile>
+ <!-- A series of build profiles where customizations for particular Hadoop releases can be made -->
+
<profile>
- <id>yarn</id>
+ <id>hadoop-0.23</id>
+ <!-- SPARK-1121: Adds an explicit dependency on Avro to work around a Hadoop 0.23.X issue -->
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.avro</groupId>
+ <artifactId>avro</artifactId>
+ </dependency>
+ </dependencies>
+ </profile>
+
+ <profile>
+ <id>hadoop-2.2</id>
+ <properties>
+ <protobuf.version>2.5.0</protobuf.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>hadoop-2.3</id>
<properties>
- <hadoop.major.version>2</hadoop.major.version>
- <hadoop.version>2.2.0</hadoop.version>
<protobuf.version>2.5.0</protobuf.version>
+ <jets3t.version>0.9.0</jets3t.version>
</properties>
+ </profile>
+
+ <profile>
+ <id>hadoop-2.4</id>
+ <properties>
+ <protobuf.version>2.5.0</protobuf.version>
+ <jets3t.version>0.9.0</jets3t.version>
+ </properties>
+ </profile>
+
+ <profile>
+ <id>yarn-alpha</id>
<modules>
<module>yarn</module>
</modules>
+ </profile>
+ <profile>
+ <id>yarn</id>
+ <modules>
+ <module>yarn</module>
+ </modules>
</profile>
<!-- Build without Hadoop dependencies that are included in some runtime environments. -->
diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 19aa3c0607a2105ca77e8dc77d5015d8c6eb0fdc..a2597e3e6ddd64edee1b93014fb5732661dfaff4 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -95,7 +95,7 @@ object SparkBuild extends Build {
lazy val hadoopVersion = Properties.envOrElse("SPARK_HADOOP_VERSION", DEFAULT_HADOOP_VERSION)
lazy val isNewHadoop = Properties.envOrNone("SPARK_IS_NEW_HADOOP") match {
case None => {
- val isNewHadoopVersion = "2.[2-9]+".r.findFirstIn(hadoopVersion).isDefined
+ val isNewHadoopVersion = "^2\\.[2-9]+".r.findFirstIn(hadoopVersion).isDefined
(isNewHadoopVersion|| DEFAULT_IS_NEW_HADOOP)
}
case Some(v) => v.toBoolean
@@ -297,6 +297,7 @@ object SparkBuild extends Build {
val chillVersion = "0.3.6"
val codahaleMetricsVersion = "3.0.0"
val jblasVersion = "1.2.3"
+ val jets3tVersion = if ("^2\\.[3-9]+".r.findFirstIn(hadoopVersion).isDefined) "0.9.0" else "0.7.1"
val jettyVersion = "8.1.14.v20131031"
val hiveVersion = "0.12.0"
val parquetVersion = "1.3.2"
@@ -343,7 +344,7 @@ object SparkBuild extends Build {
"colt" % "colt" % "1.2.0",
"org.apache.mesos" % "mesos" % "0.13.0",
"commons-net" % "commons-net" % "2.2",
- "net.java.dev.jets3t" % "jets3t" % "0.7.1" excludeAll(excludeCommonsLogging),
+ "net.java.dev.jets3t" % "jets3t" % jets3tVersion excludeAll(excludeCommonsLogging),
"org.apache.derby" % "derby" % "10.4.2.0" % "test",
"org.apache.hadoop" % hadoopClient % hadoopVersion excludeAll(excludeNetty, excludeAsm, excludeCommonsLogging, excludeSLF4J, excludeOldAsm),
"org.apache.curator" % "curator-recipes" % "2.4.0" excludeAll(excludeNetty),