diff --git a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala index 134624c35a57e4f63302753e903dca4768db1679..fb9cc116cd08b5b8051c1f1947e830e001d54759 100755 --- a/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala +++ b/core/src/main/scala/org/apache/spark/deploy/worker/Worker.scala @@ -65,7 +65,7 @@ private[spark] class Worker( val REGISTRATION_TIMEOUT = 20.seconds val REGISTRATION_RETRIES = 3 - val CLEANUP_ENABLED = conf.getBoolean("spark.worker.cleanup.enabled", true) + val CLEANUP_ENABLED = conf.getBoolean("spark.worker.cleanup.enabled", false) // How often worker will clean up old app folders val CLEANUP_INTERVAL_MILLIS = conf.getLong("spark.worker.cleanup.interval", 60 * 30) * 1000 // TTL for app folders/data; after TTL expires it will be cleaned up diff --git a/docs/configuration.md b/docs/configuration.md index 2eed96f704a4fbf7dcd9c033919368c3f3b95de0..f89040d714773abd9b5125ad8b5d941bdbe49ea9 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -390,10 +390,11 @@ Apart from these, the following properties are also available, and may be useful </tr> <tr> <td>spark.worker.cleanup.enabled</td> - <td>true</td> + <td>false</td> <td> Enable periodic cleanup of worker / application directories. Note that this only affects standalone - mode, as YARN works differently. + mode, as YARN works differently. Applications directories are cleaned up regardless of whether + the application is still running. </td> </tr> <tr>