Repository: spark Updated Branches: refs/heads/master e083db2e9 -> 6747171eb
[SPARK-15067][YARN] YARN executors are launched with fixed perm gen size ## What changes were proposed in this pull request? Look for MaxPermSize arguments anywhere in an arg, to account for quoted args. See JIRA for discussion. ## How was this patch tested? Jenkins tests Author: Sean Owen <[email protected]> Closes #12985 from srowen/SPARK-15067. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6747171e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6747171e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6747171e Branch: refs/heads/master Commit: 6747171eb19dec57c4076cab32580e42ffeb4f51 Parents: e083db2 Author: Sean Owen <[email protected]> Authored: Mon May 9 11:10:36 2016 -0700 Committer: Andrew Or <[email protected]> Committed: Mon May 9 11:10:36 2016 -0700 ---------------------------------------------------------------------- .../spark/launcher/CommandBuilderUtils.java | 2 +- .../launcher/CommandBuilderUtilsSuite.java | 40 +++++++++++++++++++- 2 files changed, 39 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/6747171e/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java ---------------------------------------------------------------------- diff --git a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java index 91586aa..62a2200 100644 --- a/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java +++ b/launcher/src/main/java/org/apache/spark/launcher/CommandBuilderUtils.java @@ -325,7 +325,7 @@ class CommandBuilderUtils { return; } for (String arg : cmd) { - if (arg.startsWith("-XX:MaxPermSize=")) { + if (arg.contains("-XX:MaxPermSize=")) { return; } } http://git-wip-us.apache.org/repos/asf/spark/blob/6747171e/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java ---------------------------------------------------------------------- diff --git a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java index 4fafc43..caeeea5 100644 --- a/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java +++ b/launcher/src/test/java/org/apache/spark/launcher/CommandBuilderUtilsSuite.java @@ -99,12 +99,48 @@ public class CommandBuilderUtilsSuite { assertEquals(10, javaMajorVersion("10")); } - private void testOpt(String opts, List<String> expected) { + @Test + public void testAddPermGenSizeOpt() { + List<String> cmd = new ArrayList<>(); + + if (javaMajorVersion(System.getProperty("java.version")) > 7) { + // Does nothing in Java 8 + addPermGenSizeOpt(cmd); + assertEquals(0, cmd.size()); + cmd.clear(); + + } else { + addPermGenSizeOpt(cmd); + assertEquals(1, cmd.size()); + assertTrue(cmd.get(0).startsWith("-XX:MaxPermSize=")); + cmd.clear(); + + cmd.add("foo"); + addPermGenSizeOpt(cmd); + assertEquals(2, cmd.size()); + assertTrue(cmd.get(1).startsWith("-XX:MaxPermSize=")); + cmd.clear(); + + cmd.add("-XX:MaxPermSize=512m"); + addPermGenSizeOpt(cmd); + assertEquals(1, cmd.size()); + assertEquals("-XX:MaxPermSize=512m", cmd.get(0)); + cmd.clear(); + + cmd.add("'-XX:MaxPermSize=512m'"); + addPermGenSizeOpt(cmd); + assertEquals(1, cmd.size()); + assertEquals("'-XX:MaxPermSize=512m'", cmd.get(0)); + cmd.clear(); + } + } + + private static void testOpt(String opts, List<String> expected) { assertEquals(String.format("test string failed to parse: [[ %s ]]", opts), expected, parseOptionString(opts)); } - private void testInvalidOpt(String opts) { + private static void testInvalidOpt(String opts) { try { parseOptionString(opts); fail("Expected exception for invalid option string."); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
