diff --git a/assembly/pom.xml b/assembly/pom.xml
index f1f8b0d3682e2f8e862535ad1ce021a54cb67ad8..20593e710dedb60c4f83957e1ddc31ecb120be77 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -213,16 +213,6 @@
</plugins>
</build>
</profile>
- <profile>
- <id>kinesis-asl</id>
- <dependencies>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${commons.httpclient.version}</version>
- </dependency>
- </dependencies>
- </profile>
<!-- Profiles that disable inclusion of certain dependencies. -->
<profile>
diff --git a/core/pom.xml b/core/pom.xml
index e80829b7a7f3d0726cfdf6e54f32127a329faf5a..5e89d548cd47f71e0ec8b17ed86bfeebe0efc393 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -74,6 +74,10 @@
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
@@ -275,7 +279,7 @@
<dependency>
<groupId>org.tachyonproject</groupId>
<artifactId>tachyon-client</artifactId>
- <version>0.5.0</version>
+ <version>0.6.4</version>
<exclusions>
<exclusion>
<groupId>org.apache.hadoop</groupId>
diff --git a/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala b/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala
index 951897cead9960bc231f4148d965cec601a1a3ba..583f1fdf0475b87dfdc91e830c8f8c57e363c487 100644
--- a/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala
+++ b/core/src/main/scala/org/apache/spark/storage/TachyonBlockManager.scala
@@ -20,8 +20,8 @@ package org.apache.spark.storage
import java.text.SimpleDateFormat
import java.util.{Date, Random}
-import tachyon.client.TachyonFS
-import tachyon.client.TachyonFile
+import tachyon.TachyonURI
+import tachyon.client.{TachyonFile, TachyonFS}
import org.apache.spark.Logging
import org.apache.spark.executor.ExecutorExitCode
@@ -40,7 +40,7 @@ private[spark] class TachyonBlockManager(
val master: String)
extends Logging {
- val client = if (master != null && master != "") TachyonFS.get(master) else null
+ val client = if (master != null && master != "") TachyonFS.get(new TachyonURI(master)) else null
if (client == null) {
logError("Failed to connect to the Tachyon as the master address is not configured")
@@ -60,11 +60,11 @@ private[spark] class TachyonBlockManager(
addShutdownHook()
def removeFile(file: TachyonFile): Boolean = {
- client.delete(file.getPath(), false)
+ client.delete(new TachyonURI(file.getPath()), false)
}
def fileExists(file: TachyonFile): Boolean = {
- client.exist(file.getPath())
+ client.exist(new TachyonURI(file.getPath()))
}
def getFile(filename: String): TachyonFile = {
@@ -81,7 +81,7 @@ private[spark] class TachyonBlockManager(
if (old != null) {
old
} else {
- val path = tachyonDirs(dirId) + "/" + "%02x".format(subDirId)
+ val path = new TachyonURI(s"${tachyonDirs(dirId)}/${"%02x".format(subDirId)}")
client.mkdir(path)
val newDir = client.getFile(path)
subDirs(dirId)(subDirId) = newDir
@@ -89,7 +89,7 @@ private[spark] class TachyonBlockManager(
}
}
}
- val filePath = subDir + "/" + filename
+ val filePath = new TachyonURI(s"$subDir/$filename")
if(!client.exist(filePath)) {
client.createFile(filePath)
}
@@ -113,7 +113,7 @@ private[spark] class TachyonBlockManager(
tries += 1
try {
tachyonDirId = "%s-%04x".format(dateFormat.format(new Date), rand.nextInt(65536))
- val path = rootDir + "/" + "spark-tachyon-" + tachyonDirId
+ val path = new TachyonURI(s"$rootDir/spark-tachyon-$tachyonDirId")
if (!client.exist(path)) {
foundLocalDir = client.mkdir(path)
tachyonDir = client.getFile(path)
diff --git a/core/src/main/scala/org/apache/spark/util/Utils.scala b/core/src/main/scala/org/apache/spark/util/Utils.scala
index 2feb7341b159ba6fcfad8fd91d4afdeec825dfd8..667aa168e7ef34486d832f670df74def5917f114 100644
--- a/core/src/main/scala/org/apache/spark/util/Utils.scala
+++ b/core/src/main/scala/org/apache/spark/util/Utils.scala
@@ -42,6 +42,8 @@ import org.apache.hadoop.security.UserGroupInformation
import org.apache.log4j.PropertyConfigurator
import org.eclipse.jetty.util.MultiException
import org.json4s._
+
+import tachyon.TachyonURI
import tachyon.client.{TachyonFS, TachyonFile}
import org.apache.spark._
@@ -955,7 +957,7 @@ private[spark] object Utils extends Logging {
* Delete a file or directory and its contents recursively.
*/
def deleteRecursively(dir: TachyonFile, client: TachyonFS) {
- if (!client.delete(dir.getPath(), true)) {
+ if (!client.delete(new TachyonURI(dir.getPath()), true)) {
throw new IOException("Failed to delete the tachyon dir: " + dir)
}
}
diff --git a/examples/pom.xml b/examples/pom.xml
index afd7c6d52f0dd88d4190bcf5df35d73180ebf90e..df1717403b67370af1da3e191e8da7d25b38a552 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -390,11 +390,6 @@
<artifactId>spark-streaming-kinesis-asl_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.httpcomponents</groupId>
- <artifactId>httpclient</artifactId>
- <version>${commons.httpclient.version}</version>
- </dependency>
</dependencies>
</profile>
<profile>
diff --git a/launcher/pom.xml b/launcher/pom.xml
index 182e5f60218dbf4e96680ffe12385c3684dd7cc3..ebfa7685eaa1826b07f16b25e55c0ccb4381648d 100644
--- a/launcher/pom.xml
+++ b/launcher/pom.xml
@@ -68,6 +68,12 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </exclusion>
+ </exclusions>
</dependency>
</dependencies>
diff --git a/make-distribution.sh b/make-distribution.sh
index 738a9c4d69601dabc58a66393380c6236df959b4..cb65932b4abc0459a9cc03316ab1b1fbac00d7b2 100755
--- a/make-distribution.sh
+++ b/make-distribution.sh
@@ -32,7 +32,7 @@ SPARK_HOME="$(cd "`dirname "$0"`"; pwd)"
DISTDIR="$SPARK_HOME/dist"
SPARK_TACHYON=false
-TACHYON_VERSION="0.5.0"
+TACHYON_VERSION="0.6.4"
TACHYON_TGZ="tachyon-${TACHYON_VERSION}-bin.tar.gz"
TACHYON_URL="https://github.com/amplab/tachyon/releases/download/v${TACHYON_VERSION}/${TACHYON_TGZ}"
diff --git a/pom.xml b/pom.xml
index bcc2f57f1af5dbc533977c3019c8965144824120..4b0b0c85eff2107e77bfec70b441595789b9192d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -146,7 +146,7 @@
<jets3t.version>0.7.1</jets3t.version>
<aws.java.sdk.version>1.8.3</aws.java.sdk.version>
<aws.kinesis.client.version>1.1.0</aws.kinesis.client.version>
- <commons.httpclient.version>4.2.6</commons.httpclient.version>
+ <commons.httpclient.version>4.3.2</commons.httpclient.version>
<commons.math3.version>3.4.1</commons.math3.version>
<test_classpath_file>${project.build.directory}/spark-test-classpath.txt</test_classpath_file>
<scala.version>2.10.4</scala.version>
@@ -420,6 +420,16 @@
<artifactId>jsr305</artifactId>
<version>1.3.9</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${commons.httpclient.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpcore</artifactId>
+ <version>${commons.httpclient.version}</version>
+ </dependency>
<dependency>
<groupId>org.seleniumhq.selenium</groupId>
<artifactId>selenium-java</artifactId>
diff --git a/sql/hive/pom.xml b/sql/hive/pom.xml
index 04440076a26a338267ecb403f543941b01bc029f..21dce8d8a565afc1c14e74ae2389696650192a4d 100644
--- a/sql/hive/pom.xml
+++ b/sql/hive/pom.xml
@@ -59,6 +59,11 @@
<groupId>${hive.group}</groupId>
<artifactId>hive-exec</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.httpcomponents</groupId>
+ <artifactId>httpclient</artifactId>
+ <version>${commons.httpclient.version}</version>
+ </dependency>
<dependency>
<groupId>org.codehaus.jackson</groupId>
<artifactId>jackson-mapper-asl</artifactId>