Skip to content
Snippets Groups Projects
Commit b29599e5 authored by Matei Zaharia's avatar Matei Zaharia
Browse files

Fix code that depended on metadata cleaner interval being in minutes

parent a1ecec8d
No related branches found
No related tags found
No related merge requests found
...@@ -198,10 +198,10 @@ abstract class DStream[T: ClassManifest] ( ...@@ -198,10 +198,10 @@ abstract class DStream[T: ClassManifest] (
metadataCleanerDelay < 0 || rememberDuration.milliseconds < metadataCleanerDelay * 1000, metadataCleanerDelay < 0 || rememberDuration.milliseconds < metadataCleanerDelay * 1000,
"It seems you are doing some DStream window operation or setting a checkpoint interval " + "It seems you are doing some DStream window operation or setting a checkpoint interval " +
"which requires " + this.getClass.getSimpleName + " to remember generated RDDs for more " + "which requires " + this.getClass.getSimpleName + " to remember generated RDDs for more " +
"than " + rememberDuration.milliseconds + " milliseconds. But the Spark's metadata cleanup" + "than " + rememberDuration.milliseconds / 1000 + " seconds. But Spark's metadata cleanup" +
"delay is set to " + (metadataCleanerDelay / 60.0) + " minutes, which is not sufficient. Please set " + "delay is set to " + metadataCleanerDelay + " seconds, which is not sufficient. Please " +
"the Java property 'spark.cleaner.delay' to more than " + "set the Java property 'spark.cleaner.delay' to more than " +
math.ceil(rememberDuration.milliseconds.toDouble / 60000.0).toInt + " minutes." math.ceil(rememberDuration.milliseconds / 1000.0).toInt + " seconds."
) )
dependencies.foreach(_.validate()) dependencies.foreach(_.validate())
......
...@@ -389,7 +389,7 @@ object StreamingContext { ...@@ -389,7 +389,7 @@ object StreamingContext {
// Set the default cleaner delay to an hour if not already set. // Set the default cleaner delay to an hour if not already set.
// This should be sufficient for even 1 second interval. // This should be sufficient for even 1 second interval.
if (MetadataCleaner.getDelaySeconds < 0) { if (MetadataCleaner.getDelaySeconds < 0) {
MetadataCleaner.setDelaySeconds(60) MetadataCleaner.setDelaySeconds(3600)
} }
new SparkContext(master, frameworkName) new SparkContext(master, frameworkName)
} }
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment