This is an automated email from the ASF dual-hosted git repository.
zjffdu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new df86e50 [ZEPPELIN-4802]. pyspark warnings with spark-3.0.0-preview -
Sync of properties between JVM and PVM
df86e50 is described below
commit df86e50218eaede551b1a3b84d6142646a17a42c
Author: Jeff Zhang <zjf...@apache.org>
AuthorDate: Wed May 27 17:37:52 2020 +0800
[ZEPPELIN-4802]. pyspark warnings with spark-3.0.0-preview - Sync of
properties between JVM and PVM
### What is this PR for?
This PR will first set env `PYSPARK_PIN_THREAD` to be `true` and then
suppress the warning message in `pyspark.util`.
### What type of PR is it?
[Improvement ]
### Todos
* [ ] - Task
### What is the Jira issue?
* https://issues.apache.org/jira/browse/ZEPPELIN-4802
### How should this be tested?
* CI pass
* Manually tested
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? no
* Is there breaking changes for older versions? No
* Does this needs documentation?
Author: Jeff Zhang <zjf...@apache.org>
Closes #3778 from zjffdu/ZEPPELIN-4802 and squashes the following commits:
d6a65cbde [Jeff Zhang] [ZEPPELIN-4802]. pyspark warnings with
spark-3.0.0-preview - Sync of properties between JVM and PVM
---
.../src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java | 4 ++++
.../src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java | 4 ++++
spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py | 4 ++++
spark/interpreter/src/main/resources/python/zeppelin_pyspark.py | 5 +++++
.../zeppelin/interpreter/launcher/SparkInterpreterLauncher.java | 2 ++
5 files changed, 19 insertions(+)
diff --git
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
index b1e1baf..c7d449f 100644
---
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
+++
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/IPySparkInterpreter.java
@@ -148,6 +148,10 @@ public class IPySparkInterpreter extends
IPythonInterpreter {
return sparkInterpreter.getSparkVersion().getMajorVersion() == 1;
}
+ public boolean isSpark3() {
+ return sparkInterpreter.getSparkVersion().getMajorVersion() == 3;
+ }
+
public JavaSparkContext getJavaSparkContext() {
return sparkInterpreter.getJavaSparkContext();
}
diff --git
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
index f180799..945aa40 100644
---
a/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
+++
b/spark/interpreter/src/main/java/org/apache/zeppelin/spark/PySparkInterpreter.java
@@ -221,4 +221,8 @@ public class PySparkInterpreter extends PythonInterpreter {
public boolean isSpark1() {
return sparkInterpreter.getSparkVersion().getMajorVersion() == 1;
}
+
+ public boolean isSpark3() {
+ return sparkInterpreter.getSparkVersion().getMajorVersion() == 3;
+ }
}
diff --git a/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py
b/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py
index 5d199bc..4b9f67f 100644
--- a/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py
+++ b/spark/interpreter/src/main/resources/python/zeppelin_ipyspark.py
@@ -39,6 +39,10 @@ java_import(gateway.jvm, "org.apache.spark.api.python.*")
java_import(gateway.jvm, "org.apache.spark.mllib.api.python.*")
intp = gateway.entry_point
+
+if intp.isSpark3():
+ warnings.filterwarnings(action='ignore', module='pyspark.util')
+
jsc = intp.getJavaSparkContext()
java_import(gateway.jvm, "org.apache.spark.sql.*")
diff --git a/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py
b/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py
index 7132b8a..9a02cd2 100644
--- a/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py
+++ b/spark/interpreter/src/main/resources/python/zeppelin_pyspark.py
@@ -15,6 +15,8 @@
# limitations under the License.
#
+import warnings
+
from py4j.java_gateway import java_import
from pyspark.conf import SparkConf
from pyspark.context import SparkContext
@@ -24,6 +26,9 @@ from pyspark.sql import SQLContext, Row
intp = gateway.entry_point
+if intp.isSpark3():
+ warnings.filterwarnings(action='ignore', module='pyspark.util')
+
jsc = intp.getJavaSparkContext()
java_import(gateway.jvm, "org.apache.spark.SparkEnv")
java_import(gateway.jvm, "org.apache.spark.SparkConf")
diff --git
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
index 7dc888f..617587a 100644
---
a/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
+++
b/zeppelin-zengine/src/main/java/org/apache/zeppelin/interpreter/launcher/SparkInterpreterLauncher.java
@@ -187,6 +187,8 @@ public class SparkInterpreterLauncher extends
StandardInterpreterLauncher {
} else {
LOGGER.info("Run Spark under non-secure mode as no keytab and principal
is specified");
}
+
+ env.put("PYSPARK_PIN_THREAD", "true");
LOGGER.debug("buildEnvFromProperties: " + env);
return env;
