From 6747171eb19dec57c4076cab32580e42ffeb4f51 Mon Sep 17 00:00:00 2001
From: Sean Owen <sowen@cloudera.com>
Date: Mon, 9 May 2016 11:10:36 -0700
Subject: [PATCH] [SPARK-15067][YARN] YARN executors are launched with fixed
perm gen size
## What changes were proposed in this pull request?
Look for MaxPermSize arguments anywhere in an arg, to account for quoted args. See JIRA for discussion.
## How was this patch tested?
Jenkins tests
Author: Sean Owen <sowen@cloudera.com>
Closes #12985 from srowen/SPARK-15067.
---
.../spark/launcher/CommandBuilderUtils.java | 2 +-
.../launcher/CommandBuilderUtilsSuite.java | 40 ++++++++++++++++++-
2 files changed, 39 insertions(+), 3 deletions(-)
diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
index 91586aad7b..62a22008d0 100644
--- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
+++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java
@@ -325,7 +325,7 @@ class CommandBuilderUtils {
return;
}
for (String arg : cmd) {
- if (arg.startsWith("-XX:MaxPermSize=")) {
+ if (arg.contains("-XX:MaxPermSize=")) {
return;
}
}
diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
index 4fafc43ef2..caeeea5ec6 100644
--- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
+++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java
@@ -99,12 +99,48 @@ public class CommandBuilderUtilsSuite {
assertEquals(10, javaMajorVersion("10"));
}
- private void testOpt(String opts, List<String> expected) {
+ @Test
+ public void testAddPermGenSizeOpt() {
+ List<String> cmd = new ArrayList<>();
+
+ if (javaMajorVersion(System.getProperty("java.version")) > 7) {
+ // Does nothing in Java 8
+ addPermGenSizeOpt(cmd);
+ assertEquals(0, cmd.size());
+ cmd.clear();
+
+ } else {
+ addPermGenSizeOpt(cmd);
+ assertEquals(1, cmd.size());
+ assertTrue(cmd.get(0).startsWith("-XX:MaxPermSize="));
+ cmd.clear();
+
+ cmd.add("foo");
+ addPermGenSizeOpt(cmd);
+ assertEquals(2, cmd.size());
+ assertTrue(cmd.get(1).startsWith("-XX:MaxPermSize="));
+ cmd.clear();
+
+ cmd.add("-XX:MaxPermSize=512m");
+ addPermGenSizeOpt(cmd);
+ assertEquals(1, cmd.size());
+ assertEquals("-XX:MaxPermSize=512m", cmd.get(0));
+ cmd.clear();
+
+ cmd.add("'-XX:MaxPermSize=512m'");
+ addPermGenSizeOpt(cmd);
+ assertEquals(1, cmd.size());
+ assertEquals("'-XX:MaxPermSize=512m'", cmd.get(0));
+ cmd.clear();
+ }
+ }
+
+ private static void testOpt(String opts, List<String> expected) {
assertEquals(String.format("test string failed to parse: [[ %s ]]", opts),
expected, parseOptionString(opts));
}
- private void testInvalidOpt(String opts) {
+ private static void testInvalidOpt(String opts) {
try {
parseOptionString(opts);
fail("Expected exception for invalid option string.");
--
GitLab