Skip to content
Snippets Groups Projects
Commit 94bb7fd4 authored by Reynold Xin's avatar Reynold Xin
Browse files

Merge pull request #876 from mbautin/master_hadoop_rdd_conf

Make HadoopRDD's configuration accessible
parents 9e17e456 35090958
No related branches found
No related tags found
No related merge requests found
...@@ -34,7 +34,7 @@ import org.apache.hadoop.util.ReflectionUtils ...@@ -34,7 +34,7 @@ import org.apache.hadoop.util.ReflectionUtils
import spark.{Dependency, Logging, Partition, RDD, SerializableWritable, SparkContext, SparkEnv, TaskContext} import spark.{Dependency, Logging, Partition, RDD, SerializableWritable, SparkContext, SparkEnv, TaskContext}
import spark.util.NextIterator import spark.util.NextIterator
import org.apache.hadoop.conf.Configurable import org.apache.hadoop.conf.{Configuration, Configurable}
/** /**
...@@ -132,4 +132,6 @@ class HadoopRDD[K, V]( ...@@ -132,4 +132,6 @@ class HadoopRDD[K, V](
override def checkpoint() { override def checkpoint() {
// Do nothing. Hadoop RDD should not be checkpointed. // Do nothing. Hadoop RDD should not be checkpointed.
} }
def getConf: Configuration = confBroadcast.value.value
} }
...@@ -120,4 +120,7 @@ class NewHadoopRDD[K, V]( ...@@ -120,4 +120,7 @@ class NewHadoopRDD[K, V](
val theSplit = split.asInstanceOf[NewHadoopPartition] val theSplit = split.asInstanceOf[NewHadoopPartition]
theSplit.serializableHadoopSplit.value.getLocations.filter(_ != "localhost") theSplit.serializableHadoopSplit.value.getLocations.filter(_ != "localhost")
} }
def getConf: Configuration = confBroadcast.value.value
} }
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment