This is an automated email from the ASF dual-hosted git repository.
jongyoul pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new afd4c3ec10 [ZEPPELIN-5966] Support Spark 3.5 (#4665)
afd4c3ec10 is described below
commit afd4c3ec1071e7d5cb39b57aa783735789b6c085
Author: Cheng Pan <cheng...@apache.org>
AuthorDate: Tue Oct 17 08:29:08 2023 +0800
[ZEPPELIN-5966] Support Spark 3.5 (#4665)
* [ZEPPELIN-5966] Support Spark 3.5
* empty
---
.github/workflows/core.yml | 5 +++++
docs/setup/basics/how_to_build.md | 3 ++-
spark/interpreter/pom.xml | 12 ++++++++++++
.../java/org/apache/zeppelin/spark/SparkInterpreterTest.java | 4 +++-
.../main/java/org/apache/zeppelin/spark/SparkVersion.java | 4 +++-
5 files changed, 25 insertions(+), 3 deletions(-)
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 77fa3780e7..5c8a8d7720 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -411,6 +411,11 @@ jobs:
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Phadoop3
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
+ - name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python
}}
+ if: matrix.python >= '3.8'
+ run: |
+ rm -rf spark/interpreter/metastore_db
+ ./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Phadoop3
-Pintegration -DfailIfNoTests=false ${MAVEN_ARGS}
livy-0-7-with-spark-3-4-1-under-python3:
runs-on: ubuntu-20.04
diff --git a/docs/setup/basics/how_to_build.md
b/docs/setup/basics/how_to_build.md
index 32f9918c1a..15e81f8ab7 100644
--- a/docs/setup/basics/how_to_build.md
+++ b/docs/setup/basics/how_to_build.md
@@ -95,7 +95,7 @@ Set scala version (default 2.11). Available profiles are
To be noticed, the spark profiles here only affect the unit test (no need to
specify `SPARK_HOME`) of spark interpreter.
Zeppelin doesn't require you to build with different spark to make different
versions of spark work in Zeppelin.
-You can run different versions of Spark in Zeppelin as long as you specify
`SPARK_HOME`. Actually Zeppelin supports all the versions of Spark from 3.2 to
3.4.
+You can run different versions of Spark in Zeppelin as long as you specify
`SPARK_HOME`. Actually Zeppelin supports all the versions of Spark from 3.2 to
3.5.
To build with a specific Spark version or scala versions, define one or more
of the following profiles and options:
@@ -106,6 +106,7 @@ Set spark major version
Available profiles are
```
+-Pspark-3.5
-Pspark-3.4
-Pspark-3.3
-Pspark-3.2
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index f3686a6238..32a8eee144 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -511,6 +511,18 @@
</profile>
<!-- profile spark-x only affect spark version used in test -->
+ <profile>
+ <id>spark-3.5</id>
+ <properties>
+ <datanucleus.core.version>4.1.17</datanucleus.core.version>
+ <datanucleus.rdbms.version>4.1.19</datanucleus.rdbms.version>
+ <datanucleus.apijdo.version>4.2.4</datanucleus.apijdo.version>
+ <spark.version>3.5.0</spark.version>
+ <protobuf.version>3.21.12</protobuf.version>
+ <py4j.version>0.10.9.7</py4j.version>
+ </properties>
+ </profile>
+
<profile>
<id>spark-3.4</id>
<activation>
diff --git
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
index 69d884314e..efe8782946 100644
---
a/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
+++
b/spark/interpreter/src/test/java/org/apache/zeppelin/spark/SparkInterpreterTest.java
@@ -17,6 +17,7 @@
package org.apache.zeppelin.spark;
+import org.apache.commons.lang3.StringUtils;
import org.apache.zeppelin.display.AngularObjectRegistry;
import org.apache.zeppelin.display.ui.CheckBox;
import org.apache.zeppelin.display.ui.Password;
@@ -236,7 +237,8 @@ class SparkInterpreterTest {
"val df = spark.createDataFrame(Seq((1,\"a\"),(2, null)))\n" +
"df.show()", getInterpreterContext());
assertEquals(InterpreterResult.Code.SUCCESS, result.code());
- assertTrue(output.contains(
+ // SPARK-43063 changed the output of null to NULL
+ assertTrue(StringUtils.containsIgnoreCase(output,
"+---+----+\n" +
"| _1| _2|\n" +
"+---+----+\n" +
diff --git
a/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
b/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
index 219bc656d6..fe537c360f 100644
---
a/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
+++
b/spark/spark-shims/src/main/java/org/apache/zeppelin/spark/SparkVersion.java
@@ -31,8 +31,10 @@ public class SparkVersion {
public static final SparkVersion SPARK_3_5_0 =
SparkVersion.fromVersionString("3.5.0");
+ public static final SparkVersion SPARK_4_0_0 =
SparkVersion.fromVersionString("4.0.0");
+
public static final SparkVersion MIN_SUPPORTED_VERSION = SPARK_3_2_0;
- public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_3_5_0;
+ public static final SparkVersion UNSUPPORTED_FUTURE_VERSION = SPARK_4_0_0;
private int version;
private int majorVersion;
