Skip to content
Snippets Groups Projects
Commit 6112270c authored by Thomas Graves's avatar Thomas Graves
Browse files

SPARK-1203 fix saving to hdfs from yarn

Author: Thomas Graves <tgraves@apache.org>

Closes #173 from tgravescs/SPARK-1203 and squashes the following commits:

4fd5ded [Thomas Graves] adding import
964e3f7 [Thomas Graves] SPARK-1203 fix saving to hdfs from yarn
parent d55ec86d
No related branches found
No related tags found
No related merge requests found
...@@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat => NewFileOutput ...@@ -41,6 +41,7 @@ import org.apache.hadoop.mapreduce.lib.output.{FileOutputFormat => NewFileOutput
import org.apache.hadoop.mapred.SparkHadoopWriter import org.apache.hadoop.mapred.SparkHadoopWriter
import org.apache.spark._ import org.apache.spark._
import org.apache.spark.deploy.SparkHadoopUtil
import org.apache.spark.Partitioner.defaultPartitioner import org.apache.spark.Partitioner.defaultPartitioner
import org.apache.spark.SparkContext._ import org.apache.spark.SparkContext._
import org.apache.spark.partial.{BoundedDouble, PartialResult} import org.apache.spark.partial.{BoundedDouble, PartialResult}
...@@ -723,6 +724,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)]) ...@@ -723,6 +724,7 @@ class PairRDDFunctions[K: ClassTag, V: ClassTag](self: RDD[(K, V)])
if (valueClass == null) { if (valueClass == null) {
throw new SparkException("Output value class not set") throw new SparkException("Output value class not set")
} }
SparkHadoopUtil.get.addCredentials(conf)
logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", " + logDebug("Saving as hadoop file of type (" + keyClass.getSimpleName + ", " +
valueClass.getSimpleName + ")") valueClass.getSimpleName + ")")
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment