This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch branch-0.12
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/branch-0.12 by this push:
new 48ce9ce184 [ZEPPELIN-6119] Bump default Spark version to 3.5.3
48ce9ce184 is described below
commit 48ce9ce184ba18096bbee37c3d6f613d9f05544c
Author: Cheng Pan <[email protected]>
AuthorDate: Sat Oct 5 20:11:02 2024 +0800
[ZEPPELIN-6119] Bump default Spark version to 3.5.3
### What is this PR for?
Currently, Zeppelin supports Spark 3.3, 3.4(default), and 3.5. This PR
bumps the default Spark version to 3.5.3, corresponding Scala deps are upgraded
too.
### What type of PR is it?
Improvement
### Todos
### What is the Jira issue?
ZEPPELIN-6119
### How should this be tested?
Pass GHA.
### Screenshots (if appropriate)
### Questions:
* Does the license files need to update? No.
* Is there breaking changes for older versions? No.
* Does this needs documentation? No.
Closes #4862 from pan3793/ZEPPELIN-6119.
Signed-off-by: Cheng Pan <[email protected]>
(cherry picked from commit 597f50c29e7a2478c17446ccae324697e19d66d4)
Signed-off-by: Cheng Pan <[email protected]>
---
Dockerfile | 4 ++--
docs/setup/basics/how_to_build.md | 12 ++++++------
k8s/zeppelin-server.yaml | 2 +-
pom.xml | 10 ++++------
rlang/pom.xml | 2 +-
scripts/docker/zeppelin-interpreter/env_python_3_with_R.yml | 2 +-
spark/interpreter/pom.xml | 12 ++++++------
spark/pom.xml | 6 +++---
spark/scala-2.12/pom.xml | 4 ++--
spark/scala-2.13/pom.xml | 2 +-
10 files changed, 27 insertions(+), 29 deletions(-)
diff --git a/Dockerfile b/Dockerfile
index 6f1777e086..0b5baacb71 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -21,9 +21,9 @@ ENV MAVEN_OPTS="-Xms1024M -Xmx2048M
-XX:MaxMetaspaceSize=1024m -XX:-UseGCOverhea
# Allow npm and bower to run with root privileges
RUN echo "unsafe-perm=true" > ~/.npmrc && \
echo '{ "allow_root": true }' > ~/.bowerrc && \
- ./mvnw -B package -DskipTests -Pbuild-distr -Pspark-3.4 -Pinclude-hadoop
-Phadoop3 -Pspark-scala-2.12 -Pweb-classic -Pweb-dist && \
+ ./mvnw -B package -DskipTests -Pbuild-distr -Pspark-3.5 -Pinclude-hadoop
-Phadoop3 -Pspark-scala-2.12 -Pweb-classic -Pweb-dist && \
# Example with doesn't compile all interpreters
- # ./mvnw -B package -DskipTests -Pbuild-distr -Pspark-3.4 -Pinclude-hadoop
-Phadoop3 -Pspark-scala-2.12 -Pweb-classic -Pweb-dist -pl
'!groovy,!livy,!hbase,!file,!flink' && \
+ # ./mvnw -B package -DskipTests -Pbuild-distr -Pspark-3.5 -Pinclude-hadoop
-Phadoop3 -Pspark-scala-2.12 -Pweb-classic -Pweb-dist -pl
'!groovy,!livy,!hbase,!file,!flink' && \
mv
/workspace/zeppelin/zeppelin-distribution/target/zeppelin-*-bin/zeppelin-*-bin
/opt/zeppelin/ && \
# Removing stuff saves time, because docker creates a temporary layer
rm -rf ~/.m2 && \
diff --git a/docs/setup/basics/how_to_build.md
b/docs/setup/basics/how_to_build.md
index e42f60904a..bccc2143b8 100644
--- a/docs/setup/basics/how_to_build.md
+++ b/docs/setup/basics/how_to_build.md
@@ -142,11 +142,11 @@ Build examples under zeppelin-examples directory
Here are some examples with several options:
```bash
-# build with spark-3.3, spark-scala-2.12
-./mvnw clean package -Pspark-3.3 -Pspark-scala-2.12 -DskipTests
+# build with spark-3.5, spark-scala-2.12
+./mvnw clean package -Pspark-3.5 -Pspark-scala-2.12 -DskipTests
-# build with spark-3.4, spark-scala-2.13
-./mvnw clean package -Pspark-3.4 -Pspark-scala-2.13 -DskipTests
+# build with spark-3.5, spark-scala-2.13
+./mvnw clean package -Pspark-3.5 -Pspark-scala-2.13 -DskipTests
```
@@ -287,10 +287,10 @@ To package the final distribution including the
compressed archive, run:
To build a distribution with specific profiles, run:
```sh
-./mvnw clean package -Pbuild-distr -Pspark-3.4
+./mvnw clean package -Pbuild-distr -Pspark-3.5
```
-The profiles `-Pspark-3.4` can be adjusted if you wish to build to a specific
spark versions.
+The profiles `-Pspark-3.5` can be adjusted if you wish to build to a specific
spark versions.
The archive is generated under _`zeppelin-distribution/target`_ directory
diff --git a/k8s/zeppelin-server.yaml b/k8s/zeppelin-server.yaml
index 862cdc5c92..84967becac 100644
--- a/k8s/zeppelin-server.yaml
+++ b/k8s/zeppelin-server.yaml
@@ -28,7 +28,7 @@ data:
# Default value is 'local.zeppelin-project.org' while it points 127.0.0.1
and `kubectl port-forward zeppelin-server` will give localhost to connects.
# If you have your ingress controller configured to connect to
`zeppelin-server` service and have a domain name for it (with wildcard
subdomain point the same address), you can replace serviceDomain field with
your own domain.
SERVICE_DOMAIN: local.zeppelin-project.org:8080
- ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE: spark:3.4.1
+ ZEPPELIN_K8S_SPARK_CONTAINER_IMAGE: spark:3.5.3
ZEPPELIN_K8S_CONTAINER_IMAGE: zeppelin-interpreter:0.12.0-SNAPSHOT
ZEPPELIN_HOME: /opt/zeppelin
ZEPPELIN_SERVER_RPC_PORTRANGE: 12320:12320
diff --git a/pom.xml b/pom.xml
index b9c6bb9f91..733650739c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -103,9 +103,8 @@
<maven.compiler.target>${java.version}</maven.compiler.target>
<scala.version>${scala.2.12.version}</scala.version>
<scala.binary.version>2.12</scala.binary.version>
- <scala.2.11.version>2.11.12</scala.2.11.version>
- <scala.2.12.version>2.12.17</scala.2.12.version>
- <scalatest.version>3.2.15</scalatest.version>
+ <scala.2.12.version>2.12.18</scala.2.12.version>
+ <scalatest.version>3.2.16</scalatest.version>
<scalacheck.version>1.17.0</scalacheck.version>
<!-- frontend maven plugin related versions-->
@@ -184,10 +183,9 @@
<plugin.rat.version>0.13</plugin.rat.version>
<plugin.resource.version>3.1.0</plugin.resource.version>
<plugin.s3.upload.version>1.4</plugin.s3.upload.version>
- <plugin.scala.alchim31.version>4.6.3</plugin.scala.alchim31.version>
- <plugin.scala.tools.version>2.15.2</plugin.scala.tools.version>
+ <plugin.scala.alchim31.version>4.8.0</plugin.scala.alchim31.version>
<plugin.scalate.version>1.7.1</plugin.scalate.version>
- <plugin.scalatest.version>2.0.0</plugin.scalatest.version>
+ <plugin.scalatest.version>2.2.0</plugin.scalatest.version>
<plugin.scm.version>1.11.2</plugin.scm.version>
<plugin.source.version>3.2.1</plugin.source.version>
<plugin.surefire.version>3.2.2</plugin.surefire.version>
diff --git a/rlang/pom.xml b/rlang/pom.xml
index 484380de79..f70af86c8f 100644
--- a/rlang/pom.xml
+++ b/rlang/pom.xml
@@ -34,7 +34,7 @@
<properties>
<interpreter.name>r</interpreter.name>
- <spark.version>3.4.1</spark.version>
+ <spark.version>3.5.3</spark.version>
<spark.archive>spark-${spark.version}</spark.archive>
<spark.bin.download.url>
diff --git a/scripts/docker/zeppelin-interpreter/env_python_3_with_R.yml
b/scripts/docker/zeppelin-interpreter/env_python_3_with_R.yml
index 09ed9a3901..a2a9a0e2f3 100644
--- a/scripts/docker/zeppelin-interpreter/env_python_3_with_R.yml
+++ b/scripts/docker/zeppelin-interpreter/env_python_3_with_R.yml
@@ -4,7 +4,7 @@ channels:
- defaults
dependencies:
- python >=3.9,<3.10
- - pyspark=3.3.2
+ - pyspark=3.5.3
- pycodestyle
- scipy
- numpy
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index 2aa3a5eb9b..a8532c26ee 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -41,10 +41,10 @@
<wagon.version>2.7</wagon.version>
<!-- spark versions -->
- <spark.version>3.4.1</spark.version>
+ <spark.version>3.5.3</spark.version>
<protobuf.version>3.21.12</protobuf.version>
<py4j.version>0.10.9.7</py4j.version>
- <spark.scala.version>2.12.17</spark.scala.version>
+ <spark.scala.version>2.12.18</spark.scala.version>
<spark.scala.binary.version>2.12</spark.scala.binary.version>
<spark.archive>spark-${spark.version}</spark.archive>
@@ -567,7 +567,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <spark.scala.version>2.12.17</spark.scala.version>
+ <spark.scala.version>2.12.18</spark.scala.version>
<spark.scala.binary.version>2.12</spark.scala.binary.version>
</properties>
</profile>
@@ -586,7 +586,7 @@
<profile>
<id>spark-3.5</id>
<properties>
- <spark.version>3.5.0</spark.version>
+ <spark.version>3.5.3</spark.version>
<protobuf.version>3.21.12</protobuf.version>
<py4j.version>0.10.9.7</py4j.version>
</properties>
@@ -598,7 +598,7 @@
<activeByDefault>true</activeByDefault>
</activation>
<properties>
- <spark.version>3.4.1</spark.version>
+ <spark.version>3.4.3</spark.version>
<protobuf.version>3.21.12</protobuf.version>
<py4j.version>0.10.9.7</py4j.version>
</properties>
@@ -607,7 +607,7 @@
<profile>
<id>spark-3.3</id>
<properties>
- <spark.version>3.3.3</spark.version>
+ <spark.version>3.3.4</spark.version>
<protobuf.version>2.5.0</protobuf.version>
<py4j.version>0.10.9.5</py4j.version>
</properties>
diff --git a/spark/pom.xml b/spark/pom.xml
index f009741b43..3b460171ed 100644
--- a/spark/pom.xml
+++ b/spark/pom.xml
@@ -33,10 +33,10 @@
<description>Zeppelin Spark Support</description>
<properties>
- <spark.version>3.4.1</spark.version>
- <protobuf.version>2.5.0</protobuf.version>
+ <spark.version>3.5.3</spark.version>
+ <protobuf.version>3.21.12</protobuf.version>
<py4j.version>0.10.9.7</py4j.version>
- <spark.scala.version>2.12.17</spark.scala.version>
+ <spark.scala.version>2.12.18</spark.scala.version>
<spark.scala.binary.version>2.12</spark.scala.binary.version>
<scala.compile.version>${spark.scala.version}</scala.compile.version>
diff --git a/spark/scala-2.12/pom.xml b/spark/scala-2.12/pom.xml
index 15e22b4f38..1584c1dd03 100644
--- a/spark/scala-2.12/pom.xml
+++ b/spark/scala-2.12/pom.xml
@@ -31,8 +31,8 @@
<name>Zeppelin: Spark Interpreter Scala_2.12</name>
<properties>
- <spark.version>3.4.1</spark.version>
- <spark.scala.version>2.12.17</spark.scala.version>
+ <spark.version>3.5.3</spark.version>
+ <spark.scala.version>2.12.18</spark.scala.version>
<spark.scala.binary.version>2.12</spark.scala.binary.version>
<spark.scala.compile.version>${spark.scala.version}</spark.scala.compile.version>
</properties>
diff --git a/spark/scala-2.13/pom.xml b/spark/scala-2.13/pom.xml
index b088a1168c..03eb8d4118 100644
--- a/spark/scala-2.13/pom.xml
+++ b/spark/scala-2.13/pom.xml
@@ -31,7 +31,7 @@
<name>Zeppelin: Spark Interpreter Scala_2.13</name>
<properties>
- <spark.version>3.4.1</spark.version>
+ <spark.version>3.5.3</spark.version>
<spark.scala.version>2.13.8</spark.scala.version>
<spark.scala.binary.version>2.13</spark.scala.binary.version>
<spark.scala.compile.version>${spark.scala.version}</spark.scala.compile.version>
