This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 0afdb34 [ZEPPELIN-4605]. Fail to launch spark interpreter in
yarn-cluster mode because missing of zeppelin-interpreter-shaded
0afdb34 is described below
commit 0afdb34b7235215476e301f72fb3f0226c02181f
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Tue Feb 11 16:05:25 2020 +0800
[ZEPPELIN-4605]. Fail to launch spark interpreter in yarn-cluster mode
because missing of zeppelin-interpreter-shaded
# What is this PR for?
This PR would add zeppelin-interpreter-shaded jar expclitly in
SparkInterpreterLauncher in yarn cluster mode.
### What type of PR is it?
[Bug Fix]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4605
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zjf...@apache.org>
Closes #3638 from zjffdu/ZEPPELIN-4605 and squashes the following commits:
0ecb0528c [Jeff Zhang] [ZEPPELIN-4605]. Fail to launch spark interpreter in
yarn-cluster mode because missing of zeppelin-interpreter-shaded
---
.../interpreter/launcher/SparkInterpreterLauncher.java | 18 ++++++++++++++++++
.../launcher/SparkInterpreterLauncherTest.java | 9 ++++++---
2 files changed, 24 insertions(+), 3 deletions(-)
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 2da4642..5252eeb 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -120,6 +120,24 @@ public class SparkInterpreterLauncher extends
StandardInterpreterLauncher {
.map(jar ->
jar.toAbsolutePath().toString()).collect(Collectors.toList());
additionalJars.addAll(scalaJars);
+ // add zeppelin-interpreter-shaded
+ Path interpreterFolder = Paths.get(zConf.getZeppelinHome(),
"/interpreter");
+ List<String> interpreterJars = StreamSupport.stream(
+ Files.newDirectoryStream(interpreterFolder, entry ->
Files.isRegularFile(entry))
+ .spliterator(),
+ false)
+ .filter(jar ->
jar.toFile().getName().startsWith("zeppelin-interpreter-shaded")
+ && jar.toFile().getName().endsWith(".jar"))
+ .map(jar -> jar.toAbsolutePath().toString())
+ .collect(Collectors.toList());
+ if (interpreterJars.size() == 0) {
+ throw new IOException("zeppelin-interpreter-shaded jar is not
found");
+ } else if (interpreterJars.size() > 1) {
+ throw new IOException("more than 1 zeppelin-interpreter-shaded jars
are found: "
+ + StringUtils.join(interpreterJars, ","));
+ }
+ additionalJars.addAll(interpreterJars);
+
if (sparkProperties.containsKey("spark.jars")) {
sparkProperties.put("spark.jars",
sparkProperties.getProperty("spark.jars") + "," +
StringUtils.join(additionalJars, ","));
diff --git
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
index d973713..8560ba1 100644
---
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
+++
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncherTest.java
@@ -199,7 +199,8 @@ public class SparkInterpreterLauncherTest {
assertEquals("true",
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
String sparkJars = "jar_1," +
- zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" +
Util.getVersion() + ".jar";
+ zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" +
Util.getVersion() + ".jar," +
+ zeppelinHome + "/interpreter/zeppelin-interpreter-shaded-" +
Util.getVersion() + ".jar";
String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
String sparkFiles = "file_1," + zeppelinHome +
"/conf/log4j_yarn_cluster.properties";
assertEquals(InterpreterLauncher.escapeSpecialCharacter(" --master
yarn-cluster --conf spark.yarn.dist.archives=" + sparkrZip +
@@ -242,7 +243,8 @@ public class SparkInterpreterLauncherTest {
assertEquals("true",
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
String sparkJars = "jar_1," +
Paths.get(localRepoPath.toAbsolutePath().toString(),
"test.jar").toString() + "," +
- zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" +
Util.getVersion() + ".jar";
+ zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" +
Util.getVersion() + ".jar," +
+ zeppelinHome + "/interpreter/zeppelin-interpreter-shaded-" +
Util.getVersion() + ".jar";
String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
String sparkFiles = "file_1," + zeppelinHome +
"/conf/log4j_yarn_cluster.properties";
assertEquals(InterpreterLauncher.escapeSpecialCharacter(" --master yarn
--conf spark.yarn.dist.archives=" + sparkrZip +
@@ -286,7 +288,8 @@ public class SparkInterpreterLauncherTest {
assertEquals("true",
interpreterProcess.getEnv().get("ZEPPELIN_SPARK_YARN_CLUSTER"));
String sparkJars = "jar_1," +
- zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" +
Util.getVersion() + ".jar";
+ zeppelinHome + "/interpreter/spark/scala-2.11/spark-scala-2.11-" +
Util.getVersion() + ".jar," +
+ zeppelinHome + "/interpreter/zeppelin-interpreter-shaded-" +
Util.getVersion() + ".jar";
String sparkrZip = sparkHome + "/R/lib/sparkr.zip#sparkr";
// escape special characters
String sparkFiles = "{}," + zeppelinHome +
"/conf/log4j_yarn_cluster.properties";
