diff --git a/pom.xml b/pom.xml index 6906ad2d3246696b3b47efbca19b51d24b2ca66b..aee07c21389e1847ec924c8553e2633df12cfa04 100644 --- a/pom.xml +++ b/pom.xml @@ -738,39 +738,39 @@ </dependencyManagement> </profile> - <profile> - <id>new-yarn</id> - <properties> - <akka.group>org.spark-project</akka.group> - <akka.version>2.0.5-protobuf-2.5-java-1.5</akka.version> - <hadoop.major.version>2</hadoop.major.version> - <hadoop.version>2.2.0</hadoop.version> - <protobuf.version>2.5.0</protobuf.version> - </properties> + <!-- <profile> --> + <!-- <id>new-yarn</id> --> + <!-- <properties> --> + <!-- <akka.group>org.spark-project</akka.group> --> + <!-- <akka.version>2.0.5-protobuf-2.5-java-1.5</akka.version> --> + <!-- <hadoop.major.version>2</hadoop.major.version> --> + <!-- <hadoop.version>2.2.0</hadoop.version> --> + <!-- <protobuf.version>2.5.0</protobuf.version> --> + <!-- </properties> --> - <modules> - <module>new-yarn</module> - </modules> + <!-- <modules> --> + <!-- <module>new-yarn</module> --> + <!-- </modules> --> - <repositories> - <repository> - <id>maven-root</id> - <name>Maven root repository</name> - <url>http://repo1.maven.org/maven2/</url> - <releases> - <enabled>true</enabled> - </releases> - <snapshots> - <enabled>false</enabled> - </snapshots> - </repository> - </repositories> + <!-- <repositories> --> + <!-- <repository> --> + <!-- <id>maven-root</id> --> + <!-- <name>Maven root repository</name> --> + <!-- <url>http://repo1.maven.org/maven2/</url> --> + <!-- <releases> --> + <!-- <enabled>true</enabled> --> + <!-- </releases> --> + <!-- <snapshots> --> + <!-- <enabled>false</enabled> --> + <!-- </snapshots> --> + <!-- </repository> --> + <!-- </repositories> --> - <dependencyManagement> - <dependencies> - </dependencies> - </dependencyManagement> - </profile> + <!-- <dependencyManagement> --> + <!-- <dependencies> --> + <!-- </dependencies> --> + <!-- </dependencyManagement> --> + <!-- </profile> --> <profile> <id>repl-bin</id> diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala index 05d3f18ea219e93b6216050e84c4471a71a6abac..c322cbd6b5d08682b138c11442b2c9732733d89c 100644 --- a/project/SparkBuild.scala +++ b/project/SparkBuild.scala @@ -78,18 +78,28 @@ object SparkBuild extends Build { } case Some(v) => v.toBoolean } + lazy val isYarnEnabled = scala.util.Properties.envOrNone("SPARK_YARN") match { case None => DEFAULT_YARN case Some(v) => v.toBoolean } + if(isNewHadoop && isYarnEnabled) { + println("""Yarn with Hadoop version 2.2.x is not yet expected to work. + Please set env SPARK_HADOOP_VERSION to appropriate version or set SPARK_YARN to false.""") + throw new Exception("Yarn with Hadoop version 2.2.x is not yet expected to work.") + } + // Build against a protobuf-2.5 compatible Akka if Hadoop 2 is used. - lazy val protobufVersion = if (isNewHadoop) "2.5.0" else "2.4.1" - lazy val akkaVersion = if (isNewHadoop) "2.0.5-protobuf-2.5-java-1.5" else "2.0.5" - lazy val akkaGroup = if (isNewHadoop) "org.spark-project" else "com.typesafe.akka" + // lazy val protobufVersion = if (isNewHadoop) "2.5.0" else "2.4.1" + // lazy val akkaVersion = if (isNewHadoop) "2.0.5-protobuf-2.5-java-1.5" else "2.0.5" + // lazy val akkaGroup = if (isNewHadoop) "org.spark-project" else "com.typesafe.akka" // Conditionally include the yarn sub-project - lazy val yarn = Project("yarn", file(if (isNewHadoop) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core) + //lazy val yarn = Project("yarn", file(if (isNewHadoop) "new-yarn" else "yarn"), settings = yarnSettings) dependsOn(core) + + lazy val yarn = Project("yarn", file("yarn"), settings = yarnSettings) dependsOn(core) + lazy val maybeYarn = if (isYarnEnabled) Seq[ClasspathDependency](yarn) else Seq[ClasspathDependency]() lazy val maybeYarnRef = if (isYarnEnabled) Seq[ProjectReference](yarn) else Seq[ProjectReference]() @@ -226,8 +236,8 @@ object SparkBuild extends Build { "org.xerial.snappy" % "snappy-java" % "1.0.5", "org.ow2.asm" % "asm" % "4.0", "com.google.protobuf" % "protobuf-java" % "2.4.1", - akkaGroup %% "akka-remote" % "2.2.3" excludeAll(excludeNetty), - akkaGroup %% "akka-slf4j" % "2.2.3" excludeAll(excludeNetty), + "com.typesafe.akka" %% "akka-remote" % "2.2.3" excludeAll(excludeNetty), + "com.typesafe.akka" %% "akka-slf4j" % "2.2.3" excludeAll(excludeNetty), "net.liftweb" %% "lift-json" % "2.5.1" excludeAll(excludeNetty), "it.unimi.dsi" % "fastutil" % "6.4.4", "colt" % "colt" % "1.2.0", @@ -311,7 +321,7 @@ object SparkBuild extends Build { "org.eclipse.paho" % "mqtt-client" % "0.4.0", "com.github.sgroschupf" % "zkclient" % "0.1" excludeAll(excludeNetty), "org.twitter4j" % "twitter4j-stream" % "3.0.3" excludeAll(excludeNetty), - akkaGroup %% "akka-zeromq" % "2.2.3" excludeAll(excludeNetty) + "com.typesafe.akka" %% "akka-zeromq" % "2.2.3" excludeAll(excludeNetty) ) )