diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh
index b0218531e9eb8bad9406a271e99f5f49e8ac39c2..70ac71e45db4b08f2cef8d11aa5d4deeccc8735b 100755
--- a/bin/compute-classpath.sh
+++ b/bin/compute-classpath.sh
@@ -44,6 +44,7 @@ if [ -f "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar ]; then
CLASSPATH="$CLASSPATH:$FWDIR/sql/catalyst/target/scala-$SCALA_VERSION/classes"
CLASSPATH="$CLASSPATH:$FWDIR/sql/core/target/scala-$SCALA_VERSION/classes"
CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SCALA_VERSION/classes"
+ CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SCALA_VERSION/classes"
DEPS_ASSEMBLY_JAR=`ls "$ASSEMBLY_DIR"/spark-assembly*hadoop*-deps.jar`
CLASSPATH="$CLASSPATH:$DEPS_ASSEMBLY_JAR"
diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md
index a5e53034674019330f9d28fec5cec3df39739324..e447dfea3bac9715d4502bc94fd1441040ddf410 100644
--- a/docs/building-with-maven.md
+++ b/docs/building-with-maven.md
@@ -45,17 +45,20 @@ For Apache Hadoop versions 1.x, Cloudera CDH MRv1, and other Hadoop versions wit
For Apache Hadoop 2.x, 0.23.x, Cloudera CDH MRv2, and other Hadoop versions with YARN, you can enable the "yarn-alpha" or "yarn" profile and set the "hadoop.version", "yarn.version" property. Note that Hadoop 0.23.X requires a special `-Phadoop-0.23` profile:
# Apache Hadoop 2.0.5-alpha
- $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -Dyarn.version=2.0.5-alpha -DskipTests clean package
+ $ mvn -Pyarn-alpha -Dhadoop.version=2.0.5-alpha -DskipTests clean package
# Cloudera CDH 4.2.0 with MapReduce v2
- $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -Dyarn.version=2.0.0-cdh4.2.0 -DskipTests clean package
+ $ mvn -Pyarn-alpha -Dhadoop.version=2.0.0-cdh4.2.0 -DskipTests clean package
# Apache Hadoop 2.2.X (e.g. 2.2.0 as below) and newer
- $ mvn -Pyarn -Dhadoop.version=2.2.0 -Dyarn.version=2.2.0 -DskipTests clean package
+ $ mvn -Pyarn -Dhadoop.version=2.2.0 -DskipTests clean package
# Apache Hadoop 0.23.x
$ mvn -Pyarn-alpha -Phadoop-0.23 -Dhadoop.version=0.23.7 -Dyarn.version=0.23.7 -DskipTests clean package
+ # Different versions of HDFS and YARN.
+ $ mvn -Pyarn-alpha -Dhadoop.version=2.3.0 -Dyarn.version=0.23.7 -DskipTests clean package
+
## Spark Tests in Maven ##
Tests are run by default via the [ScalaTest Maven plugin](http://www.scalatest.org/user_guide/using_the_scalatest_maven_plugin). Some of the require Spark to be packaged first, so always run `mvn package` with `-DskipTests` the first time. You can then run the tests with `mvn -Dhadoop.version=... test`.
diff --git a/pom.xml b/pom.xml
index 646753fe30301ccf55b4c205866724c7a24781f7..ebd359a9de17f31117b6d6e9f68767d28458e955 100644
--- a/pom.xml
+++ b/pom.xml
@@ -16,7 +16,8 @@
~ limitations under the License.
-->
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.apache</groupId>
@@ -119,7 +120,7 @@
<log4j.version>1.2.17</log4j.version>
<hadoop.version>1.0.4</hadoop.version>
<protobuf.version>2.4.1</protobuf.version>
- <yarn.version>0.23.7</yarn.version>
+ <yarn.version>${hadoop.version}</yarn.version>
<hbase.version>0.94.6</hbase.version>
<hive.version>0.12.0</hive.version>
<parquet.version>1.3.2</parquet.version>
@@ -135,7 +136,8 @@
<repositories>
<repository>
- <id>maven-repo</id> <!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
+ <id>maven-repo</id>
+ <!-- This should be at top, it makes maven try the central repo first and then others and hence faster dep resolution -->
<name>Maven Repository</name>
<!-- HTTPS is unavailable for Maven Central -->
<url>http://repo.maven.apache.org/maven2</url>
@@ -847,15 +849,16 @@
<hadoop.version>0.23.7</hadoop.version>
<!--<hadoop.version>2.0.5-alpha</hadoop.version> -->
</properties>
- <modules>
- <module>yarn</module>
- </modules>
<dependencies>
<dependency>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</dependency>
</dependencies>
+ <modules>
+ <module>yarn</module>
+ </modules>
+
</profile>
<!-- Ganglia integration is not included by default due to LGPL-licensed code -->
diff --git a/yarn/pom.xml b/yarn/pom.xml
index 02f36627431b921b802eb1e0f49cf8dd3281d35d..e0094738266b14262c2452f47bd69a1fc73ddc3a 100644
--- a/yarn/pom.xml
+++ b/yarn/pom.xml
@@ -28,7 +28,7 @@
<artifactId>yarn-parent_2.10</artifactId>
<packaging>pom</packaging>
<name>Spark Project YARN Parent POM</name>
-
+
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
@@ -50,7 +50,6 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>${yarn.version}</version>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
@@ -128,7 +127,7 @@
<target>
<property name="spark.classpath" refid="maven.test.classpath" />
<property environment="env" />
- <fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry.">
+ <fail message="Please set the SCALA_HOME (or SCALA_LIBRARY_PATH if scala is on the path) environment variables and retry.">
<condition>
<not>
<or>