diff --git a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala index be8f6529f7a1cd18d1c1129ccc302be249f402b6..c22bb8d9c60a9c8897593d05e1e31aa6fdd59cb9 100644 --- a/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala +++ b/core/src/main/scala/org/apache/spark/util/collection/ExternalAppendOnlyMap.scala @@ -74,7 +74,7 @@ class ExternalAppendOnlyMap[K, V, C]( // Collective memory threshold shared across all running tasks private val maxMemoryThreshold = { - val memoryFraction = sparkConf.getDouble("spark.shuffle.memoryFraction", 0.3) + val memoryFraction = sparkConf.getDouble("spark.shuffle.memoryFraction", 0.2) val safetyFraction = sparkConf.getDouble("spark.shuffle.safetyFraction", 0.8) (Runtime.getRuntime.maxMemory * memoryFraction * safetyFraction).toLong } diff --git a/docs/configuration.md b/docs/configuration.md index 4e4b78153a105e2a5ee65e886b51aaf96f296de1..46e3dd914b5acf66c0dac2c140556a1d5a22276c 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -239,7 +239,7 @@ Apart from these, the following properties are also available, and may be useful </tr> <tr> <td><code>spark.shuffle.memoryFraction</code></td> - <td>0.3</td> + <td>0.2</td> <td> Fraction of Java heap to use for aggregation and cogroups during shuffles, if <code>spark.shuffle.spill</code> is true. At any given time, the collective size of