This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 3fdc590 [ZEPPELIN-4497] Change zeppelin-jupyter-adapter to
zeppelin-jupyter-interpreter
3fdc590 is described below
commit 3fdc590cdcf7388997e17f6a0b2283f5b54c1bf1
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Fri Dec 20 22:52:21 2019 +0800
[ZEPPELIN-4497] Change zeppelin-jupyter-adapter to
zeppelin-jupyter-interpreter
### What is this PR for?
In ZEPPELIN-4480, we introduce zeppelin-jupyter-adapter which build a
bridge between zeppelin and jupyter, zeppelin can extends this class to
leverage any kernel of jupyter.
After a second thought, I think it is not enough. We can allow user to use
jupyter kernel directly instead of implement one by one. So the purpose is to
allow user use jupyter kernel like this
```
%jupyter(kernel=ipython)
python_code
...
```
Of course user can enhance the jupyter kernel in Zeppelin, e.g.
IPythonInterpreter of Zeppelin can enhance the ipython kernel by adding support
of ZeppelinContext
### What type of PR is it?
[Refactoring]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://jira.apache.org/jira/browse/ZEPPELIN-4497#
### How should this be tested?
* CI pass
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Jeff Zhang <zjf...@apache.org>
Closes #3558 from zjffdu/ZEPPELIN-4497 and squashes the following commits:
2efdc1e0e [Jeff Zhang] [ZEPPELIN-4497] Change zeppelin-jupyter-adapter to
zeppelin-jupyter-interpreter
---
pom.xml | 2 +-
python/pom.xml | 2 +-
.../apache/zeppelin/python/IPythonInterpreter.java | 6 +-
.../pom.xml | 16 ++-
.../zeppelin/jupyter/JupyterInterpreter.java | 112 +++++++++++++++++++++
.../zeppelin/jupyter}/JupyterKernelClient.java | 2 +-
.../jupyter}/JupyterKernelInterpreter.java | 29 ++++--
.../zeppelin/jupyter/JupyterZeppelinContext.java | 46 +++++++++
.../src/main/proto/kernel.proto | 0
.../src/main/resources/grpc/generate_rpc.sh | 0
.../main/resources/grpc/jupyter/kernel_client.py | 0
.../src/main/resources/grpc/jupyter/kernel_pb2.py | 0
.../main/resources/grpc/jupyter/kernel_pb2_grpc.py | 0
.../main/resources/grpc/jupyter/kernel_server.py | 0
.../src/main/resources/interpreter-setting.json | 14 +++
15 files changed, 214 insertions(+), 15 deletions(-)
diff --git a/pom.xml b/pom.xml
index 130b01a..96a2912 100644
--- a/pom.xml
+++ b/pom.xml
@@ -58,7 +58,7 @@
<module>zeppelin-zengine</module>
<module>zeppelin-display</module>
<module>rlang</module>
- <module>zeppelin-jupyter-adapter</module>
+ <module>zeppelin-jupyter-interpreter</module>
<module>kotlin</module>
<module>groovy</module>
<module>spark</module>
diff --git a/python/pom.xml b/python/pom.xml
index a167069..76fc0d5 100644
--- a/python/pom.xml
+++ b/python/pom.xml
@@ -43,7 +43,7 @@
<dependencies>
<dependency>
<groupId>org.apache.zeppelin</groupId>
- <artifactId>zeppelin-jupyter-adapter</artifactId>
+ <artifactId>zeppelin-jupyter-interpreter</artifactId>
<version>${project.version}</version>
</dependency>
diff --git
a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
index d6dcec0..3a42624 100644
--- a/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
+++ b/python/src/main/java/org/apache/zeppelin/python/IPythonInterpreter.java
@@ -22,11 +22,11 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.zeppelin.interpreter.BaseZeppelinContext;
import org.apache.zeppelin.interpreter.InterpreterException;
-import org.apache.zeppelin.interpreter.JupyterKernelInterpreter;
import org.apache.zeppelin.interpreter.jupyter.proto.ExecuteRequest;
import org.apache.zeppelin.interpreter.jupyter.proto.ExecuteResponse;
import org.apache.zeppelin.interpreter.jupyter.proto.ExecuteStatus;
import org.apache.zeppelin.interpreter.remote.RemoteInterpreterUtils;
+import org.apache.zeppelin.jupyter.JupyterKernelInterpreter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import py4j.GatewayServer;
@@ -39,7 +39,9 @@ import java.util.Map;
import java.util.Properties;
/**
- * IPython Interpreter for Zeppelin
+ * IPython Interpreter for Zeppelin. It enhances the JupyterKernelInterpreter
by setting up
+ * communication between JVM and Python process via py4j. So that in
IPythonInterpreter
+ * you can use ZeppelinContext.
*/
public class IPythonInterpreter extends JupyterKernelInterpreter {
diff --git a/zeppelin-jupyter-adapter/pom.xml
b/zeppelin-jupyter-interpreter/pom.xml
similarity index 92%
rename from zeppelin-jupyter-adapter/pom.xml
rename to zeppelin-jupyter-interpreter/pom.xml
index 2b180d7..ffc8daa 100644
--- a/zeppelin-jupyter-adapter/pom.xml
+++ b/zeppelin-jupyter-interpreter/pom.xml
@@ -21,20 +21,20 @@
<modelVersion>4.0.0</modelVersion>
<parent>
- <artifactId>zeppelin</artifactId>
+ <artifactId>zeppelin-interpreter-parent</artifactId>
<groupId>org.apache.zeppelin</groupId>
<version>0.9.0-SNAPSHOT</version>
- <relativePath>../pom.xml</relativePath>
+ <relativePath>../zeppelin-interpreter-parent/pom.xml</relativePath>
</parent>
<groupId>org.apache.zeppelin</groupId>
- <artifactId>zeppelin-jupyter-adapter</artifactId>
+ <artifactId>zeppelin-jupyter-interpreter</artifactId>
<packaging>jar</packaging>
<version>0.9.0-SNAPSHOT</version>
<name>Zeppelin: Jupyter Adapter</name>
<properties>
- <interpreter.name>python</interpreter.name>
+ <interpreter.name>jupyter</interpreter.name>
<python.py4j.version>0.10.7</python.py4j.version>
<grpc.version>1.15.0</grpc.version>
</properties>
@@ -179,6 +179,14 @@
</plugin>
<plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ </plugin>
+
+ <plugin>
+ <artifactId>maven-shade-plugin</artifactId>
+ </plugin>
+
+ <plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-checkstyle-plugin</artifactId>
<configuration>
diff --git
a/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterInterpreter.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterInterpreter.java
new file mode 100644
index 0000000..b359468
--- /dev/null
+++
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterInterpreter.java
@@ -0,0 +1,112 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.jupyter;
+
+import org.apache.zeppelin.interpreter.AbstractInterpreter;
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+/**
+ * Interpreter for Jupyter kernel. It can work for any Jupyter kernel as long
as the kernel
+ * is installed. Specify the kernel name in paragraph properties.
+ * Run it via `%jupyter(kernel=ipython)`
+ *
+ */
+public class JupyterInterpreter extends AbstractInterpreter {
+
+ private Map<String, JupyterKernelInterpreter> kernelInterpreterMap = new
HashMap<>();
+
+ public JupyterInterpreter(Properties properties) {
+ super(properties);
+ }
+
+ @Override
+ public BaseZeppelinContext getZeppelinContext() {
+ return new
JupyterZeppelinContext(getInterpreterGroup().getInterpreterHookRegistry(),
1000);
+ }
+
+ @Override
+ protected InterpreterResult internalInterpret(
+ String st, InterpreterContext context) throws InterpreterException {
+ String kernel = context.getLocalProperties().get("kernel");
+ if (kernel == null) {
+ return new InterpreterResult(InterpreterResult.Code.ERROR, "No kernel is
specified");
+ }
+ JupyterKernelInterpreter kernelInterpreter = null;
+ synchronized (kernelInterpreterMap) {
+ if (kernelInterpreterMap.containsKey(kernel)) {
+ kernelInterpreter = kernelInterpreterMap.get(kernel);
+ } else {
+ kernelInterpreter = new JupyterKernelInterpreter(kernel, properties);
+ kernelInterpreter.open();
+ kernelInterpreterMap.put(kernel, kernelInterpreter);
+ }
+ }
+ return kernelInterpreter.interpret(st, context);
+ }
+
+ @Override
+ public void open() throws InterpreterException {
+ // do nothing
+ }
+
+ @Override
+ public void close() throws InterpreterException {
+ for (JupyterKernelInterpreter kernelInterpreter :
kernelInterpreterMap.values()) {
+ kernelInterpreter.close();
+ }
+ }
+
+ @Override
+ public void cancel(InterpreterContext context) throws InterpreterException {
+ String kernel = context.getLocalProperties().get("kernel");
+ if (kernel == null) {
+ throw new InterpreterException("No kernel is specified");
+ }
+ JupyterKernelInterpreter kernelInterpreter =
kernelInterpreterMap.get(kernel);
+ if (kernelInterpreter != null) {
+ throw new InterpreterException("No such interpreter: " + kernel);
+ }
+ kernelInterpreter.cancel(context);
+ }
+
+ @Override
+ public FormType getFormType() throws InterpreterException {
+ return FormType.NATIVE;
+ }
+
+ @Override
+ public int getProgress(InterpreterContext context) throws
InterpreterException {
+ String kernel = context.getLocalProperties().get("kernel");
+ if (kernel == null) {
+ throw new InterpreterException("No kernel is specified");
+ }
+ JupyterKernelInterpreter kernelInterpreter =
kernelInterpreterMap.get(kernel);
+ if (kernelInterpreter == null) {
+ throw new InterpreterException("No such interpreter: " + kernel);
+ }
+ return kernelInterpreter.getProgress(context);
+ }
+}
diff --git
a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelClient.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelClient.java
similarity index 99%
rename from
zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelClient.java
rename to
zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelClient.java
index f8ba79e..2a65b0c 100644
---
a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelClient.java
+++
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelClient.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.zeppelin.interpreter;
+package org.apache.zeppelin.jupyter;
import io.grpc.ManagedChannel;
import io.grpc.ManagedChannelBuilder;
diff --git
a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelInterpreter.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
similarity index 93%
rename from
zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelInterpreter.java
rename to
zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
index 0aedbbf..c680f61 100644
---
a/zeppelin-jupyter-adapter/src/main/java/org/apache/zeppelin/interpreter/JupyterKernelInterpreter.java
+++
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
@@ -15,7 +15,7 @@
* limitations under the License.
*/
-package org.apache.zeppelin.interpreter;
+package org.apache.zeppelin.jupyter;
import com.google.common.annotations.VisibleForTesting;
import io.grpc.ManagedChannelBuilder;
@@ -25,6 +25,11 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.exception.ExceptionUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.Interpreter;
+import org.apache.zeppelin.interpreter.InterpreterContext;
+import org.apache.zeppelin.interpreter.InterpreterException;
+import org.apache.zeppelin.interpreter.InterpreterResult;
import org.apache.zeppelin.interpreter.jupyter.proto.CancelRequest;
import org.apache.zeppelin.interpreter.jupyter.proto.CompletionRequest;
import org.apache.zeppelin.interpreter.jupyter.proto.CompletionResponse;
@@ -52,10 +57,11 @@ import java.util.Map;
import java.util.Properties;
/**
- * Jupyter Kernel adapter for Zeppelin. All the jupyter kernel could be used
by Zeppelin
- * by extending this class.
+ * Jupyter Kernel Interpreter for Zeppelin. One instance of this class
represents one
+ * Jupyter Kernel. You can enhance the jupyter kernel by extending this class.
+ * e.g. IPythonInterpreter.
*/
-public abstract class JupyterKernelInterpreter extends Interpreter {
+public class JupyterKernelInterpreter extends Interpreter {
private static final Logger LOGGER =
LoggerFactory.getLogger(JupyterKernelInterpreter.class);
@@ -63,6 +69,8 @@ public abstract class JupyterKernelInterpreter extends
Interpreter {
protected JupyterKernelClient jupyterKernelClient;
protected BaseZeppelinContext zeppelinContext;
+
+ private String kernel;
// working directory of jupyter kernel
protected File kernelWorkDir;
// python executable file for launching the jupyter kernel
@@ -71,11 +79,18 @@ public abstract class JupyterKernelInterpreter extends
Interpreter {
private InterpreterOutputStream interpreterOutput = new
InterpreterOutputStream(LOGGER);
+ public JupyterKernelInterpreter(String kernel, Properties properties) {
+ this(properties);
+ this.kernel = kernel;
+ }
+
public JupyterKernelInterpreter(Properties properties) {
super(properties);
}
- public abstract String getKernelName();
+ public String getKernelName() {
+ return this.kernel;
+ }
public List<String> getRequiredPackages() {
List<String> requiredPackages = new ArrayList<>();
@@ -85,7 +100,9 @@ public abstract class JupyterKernelInterpreter extends
Interpreter {
return requiredPackages;
}
- public abstract BaseZeppelinContext buildZeppelinContext();
+ protected BaseZeppelinContext buildZeppelinContext() {
+ return new JupyterZeppelinContext(null, 1000);
+ }
@Override
public void open() throws InterpreterException {
diff --git
a/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterZeppelinContext.java
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterZeppelinContext.java
new file mode 100644
index 0000000..22b28d4
--- /dev/null
+++
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterZeppelinContext.java
@@ -0,0 +1,46 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.zeppelin.jupyter;
+
+import org.apache.zeppelin.interpreter.BaseZeppelinContext;
+import org.apache.zeppelin.interpreter.InterpreterHookRegistry;
+
+import java.util.List;
+import java.util.Map;
+
+public class JupyterZeppelinContext extends BaseZeppelinContext {
+ public JupyterZeppelinContext(InterpreterHookRegistry hooks, int maxResult) {
+ super(hooks, maxResult);
+ }
+
+ @Override
+ public Map<String, String> getInterpreterClassMap() {
+ return null;
+ }
+
+ @Override
+ public List<Class> getSupportedClasses() {
+ return null;
+ }
+
+ @Override
+ public String showData(Object obj, int maxResult) {
+ return null;
+ }
+}
diff --git a/zeppelin-jupyter-adapter/src/main/proto/kernel.proto
b/zeppelin-jupyter-interpreter/src/main/proto/kernel.proto
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/proto/kernel.proto
rename to zeppelin-jupyter-interpreter/src/main/proto/kernel.proto
diff --git a/zeppelin-jupyter-adapter/src/main/resources/grpc/generate_rpc.sh
b/zeppelin-jupyter-interpreter/src/main/resources/grpc/generate_rpc.sh
similarity index 100%
rename from zeppelin-jupyter-adapter/src/main/resources/grpc/generate_rpc.sh
rename to zeppelin-jupyter-interpreter/src/main/resources/grpc/generate_rpc.sh
diff --git
a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_client.py
b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_client.py
similarity index 100%
rename from
zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_client.py
rename to
zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_client.py
diff --git
a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2.py
b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2.py
similarity index 100%
rename from
zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2.py
rename to
zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2.py
diff --git
a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py
b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py
similarity index 100%
rename from
zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py
rename to
zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_pb2_grpc.py
diff --git
a/zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_server.py
b/zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_server.py
similarity index 100%
rename from
zeppelin-jupyter-adapter/src/main/resources/grpc/jupyter/kernel_server.py
rename to
zeppelin-jupyter-interpreter/src/main/resources/grpc/jupyter/kernel_server.py
diff --git
a/zeppelin-jupyter-interpreter/src/main/resources/interpreter-setting.json
b/zeppelin-jupyter-interpreter/src/main/resources/interpreter-setting.json
new file mode 100644
index 0000000..19181f7
--- /dev/null
+++ b/zeppelin-jupyter-interpreter/src/main/resources/interpreter-setting.json
@@ -0,0 +1,14 @@
+[
+ {
+ "group": "jupyter",
+ "name": "jupyter",
+ "className": "org.apache.zeppelin.jupyter.JupyterInterpreter",
+ "properties": {
+ },
+ "editor": {
+ "language": "text",
+ "editOnDblClick": false,
+ "completionSupport": true
+ }
+ }
+]
