Skip to content
Snippets Groups Projects
Commit 09f3b677 authored by Stephen Haberman's avatar Stephen Haberman
Browse files

Avoid match errors when filtering for spark.hadoop settings.

parent 3a388c32
No related branches found
No related tags found
No related merge requests found
...@@ -248,8 +248,10 @@ class SparkContext( ...@@ -248,8 +248,10 @@ class SparkContext(
conf.set("fs.s3n.awsSecretAccessKey", System.getenv("AWS_SECRET_ACCESS_KEY")) conf.set("fs.s3n.awsSecretAccessKey", System.getenv("AWS_SECRET_ACCESS_KEY"))
} }
// Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar" // Copy any "spark.hadoop.foo=bar" system properties into conf as "foo=bar"
Utils.getSystemProperties.foreach { case (key, value) if key.startsWith("spark.hadoop.") => Utils.getSystemProperties.foreach { case (key, value) =>
conf.set(key.substring("spark.hadoop.".length), value) if (key.startsWith("spark.hadoop.")) {
conf.set(key.substring("spark.hadoop.".length), value)
}
} }
val bufferSize = System.getProperty("spark.buffer.size", "65536") val bufferSize = System.getProperty("spark.buffer.size", "65536")
conf.set("io.file.buffer.size", bufferSize) conf.set("io.file.buffer.size", bufferSize)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment