From 9862c614c06507aa7624208f1d7ed5bc027ca52e Mon Sep 17 00:00:00 2001
From: wangfei <scnbwf@yeah.net>
Date: Tue, 5 Aug 2014 00:51:07 -0700
Subject: [PATCH] [SPARK-1779] Throw an exception if memory fractions are not
between 0 and 1
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Author: wangfei <scnbwf@yeah.net>
Author: wangfei <wangfei1@huawei.com>
Closes #714 from scwf/memoryFraction and squashes the following commits:
6e385b9 [wangfei] Update SparkConf.scala
da6ee59 [wangfei] add configs
829a195 [wangfei] add indent
717c0ca [wangfei] updated to make more concise
fc45476 [wangfei] validate memoryfraction in sparkconf
2e79b3d [wangfei] && => ||
43621bd [wangfei] && => ||
cf38bcf [wangfei] throw IllegalArgumentException
14d18ac [wangfei] throw IllegalArgumentException
dff1f0f [wangfei] Update BlockManager.scala
764965f [wangfei] Update ExternalAppendOnlyMap.scala
a59d76b [wangfei] Throw exception when memoryFracton is out of range
7b899c2 [wangfei] ã€SPARK-1779】
---
.../main/scala/org/apache/spark/SparkConf.scala | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/core/src/main/scala/org/apache/spark/SparkConf.scala b/core/src/main/scala/org/apache/spark/SparkConf.scala
index 38700847c8..cce7a23d3b 100644
--- a/core/src/main/scala/org/apache/spark/SparkConf.scala
+++ b/core/src/main/scala/org/apache/spark/SparkConf.scala
@@ -238,6 +238,20 @@ class SparkConf(loadDefaults: Boolean) extends Cloneable with Logging {
}
}
+ // Validate memory fractions
+ val memoryKeys = Seq(
+ "spark.storage.memoryFraction",
+ "spark.shuffle.memoryFraction",
+ "spark.shuffle.safetyFraction",
+ "spark.storage.unrollFraction",
+ "spark.storage.safetyFraction")
+ for (key <- memoryKeys) {
+ val value = getDouble(key, 0.5)
+ if (value > 1 || value < 0) {
+ throw new IllegalArgumentException("$key should be between 0 and 1 (was '$value').")
+ }
+ }
+
// Check for legacy configs
sys.env.get("SPARK_JAVA_OPTS").foreach { value =>
val warning =
--
GitLab