[zeppelin] branch master updated: [ZEPPELIN-5417] Unable to set conda env in pyspark (#4147)

[zjffdu]

This is an automated email from the ASF dual-hosted git repository.

zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git


The following commit(s) were added to refs/heads/master by this push:
     new b9118fd  [ZEPPELIN-5417] Unable to set conda env in pyspark (#4147)
b9118fd is described below

commit b9118fd47e2e908c4c2ac05aa28d63fe424f91ba
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Sun Jul 4 14:55:17 2021 +0800

    [ZEPPELIN-5417] Unable to set conda env in pyspark (#4147)
    
    * [ZEPPELIN-5417] Unable to set conda env in pyspark
---
 .../apache/zeppelin/python/PythonInterpreter.java  | 28 ++++++++++++++--------
 .../apache/zeppelin/spark/PySparkInterpreter.java  | 16 +++++++------
 .../zeppelin/spark/IPySparkInterpreterTest.java    |  1 +
 .../zeppelin/jupyter/JupyterKernelInterpreter.java |  1 +
 4 files changed, 29 insertions(+), 17 deletions(-)

diff --git 
a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java 
b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
index 57ee9ac..23c0945 100644
--- a/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
+++ b/python/src/main/java/org/apache/zeppelin/python/PythonInterpreter.java
@@ -80,16 +80,24 @@ public class PythonInterpreter extends Interpreter {
   public void open() throws InterpreterException {
     // try IPythonInterpreter first
     iPythonInterpreter = getIPythonInterpreter();
-    if (getProperty("zeppelin.python.useIPython", "true").equals("true") &&
-        StringUtils.isEmpty(
-            iPythonInterpreter.checkKernelPrerequisite(getPythonExec()))) {
-      try {
-        iPythonInterpreter.open();
-        LOGGER.info("IPython is available, Use IPythonInterpreter to replace 
PythonInterpreter");
-        return;
-      } catch (Exception e) {
-        iPythonInterpreter = null;
-        LOGGER.warn("Fail to open IPythonInterpreter", e);
+    boolean useIPython = 
Boolean.parseBoolean(getProperty("zeppelin.python.useIPython", "true"));
+
+    LOGGER.info("zeppelin.python.useIPython: {}", useIPython);
+    if (useIPython) {
+      String checkKernelPrerequisiteResult = 
iPythonInterpreter.checkKernelPrerequisite(
+              getPythonExec());
+      if (StringUtils.isEmpty(checkKernelPrerequisiteResult)) {
+        try {
+          iPythonInterpreter.open();
+          LOGGER.info("IPython is available, Use IPythonInterpreter to replace 
PythonInterpreter");
+          return;
+        } catch (Exception e) {
+          iPythonInterpreter = null;
+          LOGGER.warn("Fail to open IPythonInterpreter", e);
+        }
+      } else {
+        LOGGER.info("IPython requirement is not met, 
checkKernelPrerequisiteResult: {}",
+                checkKernelPrerequisiteResult);
       }
     }
 
diff --git 
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
 
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
index 6f8c96f..ca4369d 100644
--- 
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
+++ 
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
@@ -171,18 +171,20 @@ public class PySparkInterpreter extends PythonInterpreter 
{
   // spark.pyspark.driver.python > spark.pyspark.python > 
PYSPARK_DRIVER_PYTHON > PYSPARK_PYTHON
   @Override
   protected String getPythonExec() {
-    if (!StringUtils.isBlank(getProperty("spark.pyspark.driver.python", ""))) {
-      return properties.getProperty("spark.pyspark.driver.python");
+    SparkConf sparkConf = getSparkConf();
+    if (StringUtils.isNotBlank(sparkConf.get("spark.pyspark.driver.python", 
""))) {
+      return sparkConf.get("spark.pyspark.driver.python");
     }
-    if (!StringUtils.isBlank(getProperty("spark.pyspark.python", ""))) {
-      return properties.getProperty("spark.pyspark.python");
-    }
-    if (System.getenv("PYSPARK_PYTHON") != null) {
-      return System.getenv("PYSPARK_PYTHON");
+    if (StringUtils.isNotBlank(sparkConf.get("spark.pyspark.python", ""))) {
+      return sparkConf.get("spark.pyspark.python");
     }
     if (System.getenv("PYSPARK_DRIVER_PYTHON") != null) {
       return System.getenv("PYSPARK_DRIVER_PYTHON");
     }
+    if (System.getenv("PYSPARK_PYTHON") != null) {
+      return System.getenv("PYSPARK_PYTHON");
+    }
+
     return "python";
   }
 
diff --git 
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java
 
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java
index 0589820..9fab73b 100644
--- 
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java
+++ 
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/IPySparkInterpreterTest.java
@@ -90,6 +90,7 @@ public class IPySparkInterpreterTest extends 
IPythonInterpreterTest {
     intpGroup.get("session_1").add(interpreter);
     interpreter.setInterpreterGroup(intpGroup);
 
+    pySparkInterpreter.open();
     interpreter.open();
   }
 
diff --git 
a/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
 
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
index 6c2fdc5..ea97af4 100644
--- 
a/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
+++ 
b/zeppelin-jupyter-interpreter/src/main/java/org/apache/zeppelin/jupyter/JupyterKernelInterpreter.java
@@ -153,6 +153,7 @@ public class JupyterKernelInterpreter extends 
AbstractInterpreter {
    * @return check result of checking kernel prerequisite.
    */
   public String checkKernelPrerequisite(String pythonExec) {
+    LOGGER.info("checkKernelPrerequisite using python executable: {}", 
pythonExec);
     ProcessBuilder processBuilder = new ProcessBuilder(pythonExec, "-m", 
"pip", "freeze");
     File stderrFile = null;
     File stdoutFile = null;