This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new cb6fa56 [ZEPPELIN-4903]. Add property to zeppelin-site to filter
unnecessary interpreters
cb6fa56 is described below
commit cb6fa5682c529f2a8e29d39231ec57ae89cbd107
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Tue Jun 23 11:09:45 2020 +0800
[ZEPPELIN-4903]. Add property to zeppelin-site to filter unnecessary
interpreters
### What is this PR for?
This PR introduce 2 properties `zeppelin.interpreter.include` and
`zeppelin.interpreter.exclude`. User can use these properties to filter out the
interpreters that he would like to use. Without this PR, user have to manually
delete corresponding interpreter folders or rebuild zeppelin.
### What type of PR is it?
[ Improvement]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4903
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zjf...@apache.org>
Closes #3819 from zjffdu/ZEPPELIN-4903 and squashes the following commits:
62a9fcf1e [Jeff Zhang] [ZEPPELIN-4903]. Add property to zeppelin-site to
filter unnecessary interpreters
---
conf/zeppelin-site.xml.template | 12 +++++
docs/usage/interpreter/overview.md | 6 +++
.../zeppelin/conf/ZeppelinConfiguration.java | 3 ++
.../interpreter/InterpreterSettingManager.java | 53 ++++++++++++++++++----
.../interpreter/InterpreterSettingManagerTest.java | 50 ++++++++++++++++++++
5 files changed, 115 insertions(+), 9 deletions(-)
diff --git a/conf/zeppelin-site.xml.template b/conf/zeppelin-site.xml.template
index be99433..8a47b79 100755
--- a/conf/zeppelin-site.xml.template
+++ b/conf/zeppelin-site.xml.template
@@ -62,6 +62,18 @@
</property>
<property>
+ <name>zeppelin.interpreter.include</name>
+ <value></value>
+ <description>All the inteprreters that you would like to include. You can
only specify either 'zeppelin.interpreter.include' or
'zeppelin.interpreter.exclude'. Specifying them together is not
allowed.</description>
+</property>
+
+<property>
+ <name>zeppelin.interpreter.exclude</name>
+ <value></value>
+ <description>All the inteprreters that you would like to exclude. You can
only specify either 'zeppelin.interpreter.include' or
'zeppelin.interpreter.exclude'. Specifying them together is not
allowed.</description>
+</property>
+
+<property>
<name>zeppelin.notebook.homescreen</name>
<value></value>
<description>id of notebook to be displayed in homescreen. ex) 2A94M5J1Z
Empty value displays default home screen</description>
diff --git a/docs/usage/interpreter/overview.md
b/docs/usage/interpreter/overview.md
index b1bbf60..7584abf 100644
--- a/docs/usage/interpreter/overview.md
+++ b/docs/usage/interpreter/overview.md
@@ -167,3 +167,9 @@ Here's one screenshot of how one running paragraph of flink
interpreter works.
<img
src="{{BASE_PATH}}/assets/themes/zeppelin/img/docs-img/flink_recovery.gif"
width="800px">
+
+## Choose Interpreters
+
+By default, Zeppelin will register and display all the interpreters under
folder `$ZEPPELIN_HOME/interpreters`.
+But you can configure property `zeppelin.interpreter.include` to specify what
interpreters you want to include or `zeppelin.interpreter.exclude` to specify
what interpreters you want to exclude.
+Only one of them can be specified, you can not specify them together.
diff --git
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
index dcb8619..7331a50 100644
---
a/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
+++
b/zeppelin-interpreter/src/main/java/org/apache/zeppelin/conf/ZeppelinConfiguration.java
@@ -919,6 +919,9 @@ public class ZeppelinConfiguration extends XMLConfiguration
{
ZEPPELIN_INTERPRETER_MAX_POOL_SIZE("zeppelin.interpreter.max.poolsize",
10),
ZEPPELIN_INTERPRETER_GROUP_DEFAULT("zeppelin.interpreter.group.default",
"spark"),
ZEPPELIN_INTERPRETER_OUTPUT_LIMIT("zeppelin.interpreter.output.limit",
1024 * 100),
+ ZEPPELIN_INTERPRETER_INCLUDES("zeppelin.interpreter.include", ""),
+ ZEPPELIN_INTERPRETER_EXCLUDES("zeppelin.interpreter.exclude", ""),
+
ZEPPELIN_ENCODING("zeppelin.encoding", "UTF-8"),
ZEPPELIN_NOTEBOOK_DIR("zeppelin.notebook.dir", "notebook"),
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
index 8b6a890..558d4b1 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/InterpreterSettingManager.java
@@ -26,6 +26,7 @@ import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.reflect.TypeToken;
+import java.util.Arrays;
import java.util.LinkedHashMap;
import java.util.Set;
import java.util.regex.Matcher;
@@ -80,7 +81,6 @@ import java.lang.reflect.Type;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLClassLoader;
-import java.nio.file.DirectoryStream.Filter;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
@@ -141,6 +141,8 @@ public class InterpreterSettingManager implements
NoteEventListener, ClusterEven
private ConfigStorage configStorage;
private RemoteInterpreterEventServer interpreterEventServer;
private Map<String, String> jupyterKernelLanguageMap = new HashMap<>();
+ private List<String> includesInterpreters;
+ private List<String> excludesInterpreters;
@Inject
public InterpreterSettingManager(ZeppelinConfiguration zeppelinConfiguration,
@@ -243,13 +245,18 @@ public class InterpreterSettingManager implements
NoteEventListener, ClusterEven
for (InterpreterSetting interpreterSettingTemplate :
interpreterSettingTemplates.values()) {
InterpreterSetting interpreterSetting = new
InterpreterSetting(interpreterSettingTemplate);
initInterpreterSetting(interpreterSetting);
- interpreterSettings.put(interpreterSetting.getId(),
interpreterSetting);
+ if (shouldRegister(interpreterSetting.getGroup())) {
+ interpreterSettings.put(interpreterSetting.getId(),
interpreterSetting);
+ }
}
return;
}
//TODO(zjffdu) still ugly (should move all to InterpreterInfoSaving)
for (InterpreterSetting savedInterpreterSetting :
infoSaving.interpreterSettings.values()) {
+ if (!shouldRegister(savedInterpreterSetting.getGroup())) {
+ break;
+ }
savedInterpreterSetting.setProperties(InterpreterSetting.convertInterpreterProperties(
savedInterpreterSetting.getProperties()
));
@@ -322,15 +329,44 @@ public class InterpreterSettingManager implements
NoteEventListener, ClusterEven
}
private void init() throws IOException {
+ this.includesInterpreters =
+
Arrays.asList(conf.getString(ConfVars.ZEPPELIN_INTERPRETER_INCLUDES).split(","))
+ .stream()
+ .filter(t -> !t.isEmpty())
+ .collect(Collectors.toList());
+ this.excludesInterpreters =
+
Arrays.asList(conf.getString(ConfVars.ZEPPELIN_INTERPRETER_EXCLUDES).split(","))
+ .stream()
+ .filter(t -> !t.isEmpty())
+ .collect(Collectors.toList());
+ if (!includesInterpreters.isEmpty() && !excludesInterpreters.isEmpty()) {
+ throw new IOException(String.format("%s and %s can not be specified
together, only one can be set.",
+ ConfVars.ZEPPELIN_INTERPRETER_INCLUDES.getVarName(),
+ ConfVars.ZEPPELIN_INTERPRETER_EXCLUDES.getVarName()));
+ }
loadJupyterKernelLanguageMap();
loadInterpreterSettingFromDefaultDir(true);
loadFromFile();
saveToFile();
- // must init Recovery after init of InterpreterSettingManagaer
+ // must init Recovery after init of InterpreterSettingManager
recoveryStorage.init();
}
+ /**
+ * We should only register interpreterSetting when
+ * 1. No setting in 'zeppelin.interpreter.include' and
'zeppelin.interpreter.exclude'
+ * 2. It is specified in 'zeppelin.interpreter.include'
+ * 3. It is not specified in 'zeppelin.interpreter.exclude'
+ * @param group
+ * @return
+ */
+ private boolean shouldRegister(String group) {
+ return (includesInterpreters.isEmpty() && excludesInterpreters.isEmpty())
||
+ (!includesInterpreters.isEmpty() && includesInterpreters.contains(group))
||
+ (!excludesInterpreters.isEmpty() &&
!excludesInterpreters.contains(group));
+ }
+
private void loadJupyterKernelLanguageMap() throws IOException {
String kernels =
conf.getString(ConfVars.ZEPPELIN_INTERPRETER_JUPYTER_KERNELS);
for (String kernel : kernels.split(",")) {
@@ -351,12 +387,11 @@ public class InterpreterSettingManager implements
NoteEventListener, ClusterEven
ClassLoader cl = Thread.currentThread().getContextClassLoader();
if (Files.exists(interpreterDirPath)) {
for (Path interpreterDir : Files
- .newDirectoryStream(interpreterDirPath, new Filter<Path>() {
- @Override
- public boolean accept(Path entry) throws IOException {
- return Files.exists(entry) && Files.isDirectory(entry);
- }
- })) {
+ .newDirectoryStream(interpreterDirPath,
+ entry -> Files.exists(entry)
+ && Files.isDirectory(entry)
+ && shouldRegister(entry.toFile().getName()))) {
+
String interpreterDirString = interpreterDir.toString();
/**
* Register interpreter by the following ordering
diff --git
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
index e18ac0b..d8a61a3 100644
---
a/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
+++
b/zeppelin-zengine/src/test/java/org/apache/zeppelin/interpreter/InterpreterSettingManagerTest.java
@@ -276,4 +276,54 @@ public class InterpreterSettingManagerTest extends
AbstractInterpreterTest {
assertEquals(1, interpreterSetting.getAllInterpreterGroups().size());
assertEquals(1,
interpreterSetting.getAllInterpreterGroups().get(0).getSessionNum());
}
+
+ @Test
+ public void testInterpreterInclude() throws Exception {
+ try {
+
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_INCLUDES.getVarName(),
"mock1");
+ setUp();
+
+ assertEquals(1, interpreterSettingManager.get().size());
+ assertEquals("mock1", interpreterSettingManager.get().get(0).getGroup());
+ } finally {
+
System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_INCLUDES.getVarName());
+ }
+ }
+
+ @Test
+ public void testInterpreterExclude() throws Exception {
+ try {
+
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_EXCLUDES.getVarName(),
+ "test,config_test,mock_resource_pool");
+ setUp();
+
+ assertEquals(2, interpreterSettingManager.get().size());
+ assertNotNull(interpreterSettingManager.getByName("mock1"));
+ assertNotNull(interpreterSettingManager.getByName("mock2"));
+ } finally {
+
System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_EXCLUDES.getVarName());
+ }
+ }
+
+ @Test
+ public void testInterpreterIncludeExcludeTogether() throws Exception {
+ try {
+
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_INCLUDES.getVarName(),
+ "test,");
+
System.setProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_EXCLUDES.getVarName(),
+ "config_test,mock_resource_pool");
+
+ try {
+ setUp();
+ fail("Should not able to create InterpreterSettingManager");
+ } catch (Exception e) {
+ e.printStackTrace();
+ assertEquals("zeppelin.interpreter.include and
zeppelin.interpreter.exclude can not be specified together, only one can be
set.",
+ e.getMessage());
+ }
+ } finally {
+
System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_INCLUDES.getVarName());
+
System.clearProperty(ZeppelinConfiguration.ConfVars.ZEPPELIN_INTERPRETER_EXCLUDES.getVarName());
+ }
+ }
}
