This is an automated email from the ASF dual-hosted git repository.
pan3793 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/zeppelin.git
The following commit(s) were added to refs/heads/master by this push:
new 703fe6763f [ZEPPELIN-6311] Support JDK 17
703fe6763f is described below
commit 703fe6763f6f5aefbe66004327579d8b6cfd95e6
Author: Cheng Pan <[email protected]>
AuthorDate: Mon Sep 7 13:24:03 2026 +0800
[ZEPPELIN-6311] Support JDK 17
## What is this PR for?
Enable Zeppelin to build, test, and run on JDK 17. This involves two areas
of work:
- Flink Scala version alignment
- JPMS module-access flags
Cassandra test migration is handled separately in #5412.
## What type of PR is it?
Improvement
## What is the Jira issue?
https://issues.apache.org/jira/browse/ZEPPELIN-6311
## How should this be tested?
CI passes on both JDK 11 and JDK 17 across all modules. Verified locally:
- `flink/flink-scala-2.12` module: 7 `FlinkInterpreterTest` tests pass on
both JDK 11 and JDK 17
- `bin/common.sh` syntax-checked; `JPMS_JAVA_OPTS` verified in `JAVA_OPTS`
and `JAVA_INTP_OPTS`
## Questions
- Does the licenses file need update? No
- Is there breaking changes for older versions? No
- Does this needs documentation? No
## Details
### 1. Flink: Scala 2.12.7 -> 2.12.20
Flink 1.19/1.20 binary tgz bundles `flink-scala_2.12-*.jar` which shades
`scala-library`, `scala-compiler`, and `scala-reflect` 2.12.7. Scala 2.12.7's
`JrtClassPath.asURLs()` crashes on JDK 17+ with:
```
java.lang.RuntimeException: /packages cannot be represented as URI
at java.base/jdk.internal.jrtfs.JrtPath.toUri(JrtPath.java:175)
at
scala.tools.nsc.classpath.JrtClassPath.asURLs(DirectoryClassPath.scala:204)
```
This was fixed in Scala 2.12.9
([scala/bug#11608](https://github.com/scala/bug/issues/11608)).
Changes:
- Change `flink.scala.version` from `2.12.7` to `2.12.20` in
`flink/pom.xml` (removes the `jdk17+` profile). The compile-time version must
match the runtime version due to binary incompatibility in
`Settings.usejavacp()` between 2.12.7 and 2.12.20.
- Add `maven-antrun-plugin` to `flink-1.19` and `flink-1.20` profiles that
patches the downloaded Flink tgz: strips `scala/*` classes from
`flink-scala_2.12-*.jar` via `zip -d`, then copies
`scala-library`/`scala-compiler`/`scala-reflect` 2.12.20 jars into
`FLINK_HOME/lib`. Scoped to Flink 1.x only.
- Add `patchFlinkScala()` in `DownloadUtils` for integration tests that
download Flink via `DownloadUtils.downloadFlink()` to `~/.cache/`. Guards on
`scalaVersion == "2.12"` and `flinkVersion < 2.0`.
Experimental support for Java 17 was added in Flink 1.18.
([FLINK-15736](https://issues.apache.org/jira/browse/FLINK-15736))
Flink 1.19/1.20 sticky with Scala 2.12.7 due to
[FLINK-12461](https://github.com/apache/flink/pull/14780), upgrade Scala is a
necessary step for JDK 17 support.
### 2. JPMS args and JDK 17 CI
JDK 17 enforces strong module encapsulation. Many libraries (Netty,
Mockito, etc.) require `--add-opens` flags to access internal JDK APIs. These
flags were already needed for surefire tests but were either missing or
duplicated per-module.
Assisted-by: GLM 5.2
Closes #5311 from pan3793/jdk17.
Signed-off-by: Cheng Pan <[email protected]>
---
.github/workflows/core.yml | 28 +++++-----
bin/common.cmd | 26 ++++++++-
bin/common.sh | 26 ++++++++-
flink/flink-scala-2.12/pom.xml | 64 +++++++++++++++++++++-
flink/pom.xml | 2 +-
pom.xml | 25 ++++++++-
spark/interpreter/pom.xml | 32 -----------
zeppelin-integration/pom.xml | 2 +-
zeppelin-interpreter-integration/pom.xml | 2 +-
zeppelin-server/pom.xml | 2 +-
.../org/apache/zeppelin/test/DownloadUtils.java | 44 +++++++++++++++
11 files changed, 196 insertions(+), 57 deletions(-)
diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml
index 816d9f7be9..efeb04ce1e 100644
--- a/.github/workflows/core.yml
+++ b/.github/workflows/core.yml
@@ -40,7 +40,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -90,7 +90,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
env:
INTERPRETERS:
'hbase,jdbc,file,flink-cmd,cassandra,elasticsearch,bigquery,livy,groovy,java,neo4j,sparql,mongodb,influxdb,shell'
steps:
@@ -136,7 +136,7 @@ jobs:
fail-fast: false
matrix:
python: [ 3.9 ]
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -181,7 +181,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
# user/password => root/root
- name: Start mysql
@@ -232,10 +232,12 @@ jobs:
fail-fast: false
matrix:
include:
- - python: 3.9
+ - java: 11
+ python: 3.9
flink: 119
flink-profile: "1.19"
- - python: 3.9
+ - java: 17
+ python: 3.9
flink: 120
flink-profile: "1.20"
steps:
@@ -243,11 +245,11 @@ jobs:
uses: actions/checkout@v5
- name: Tune Runner VM
uses: ./.github/actions/tune-runner-vm
- - name: Set up JDK 11
+ - name: Set up JDK ${{ matrix.java }}
uses: actions/setup-java@v5
with:
distribution: 'temurin'
- java-version: 11
+ java-version: ${{ matrix.java }}
- name: Cache local Maven repository
uses: actions/cache@v5
with:
@@ -285,7 +287,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -366,22 +368,18 @@ jobs:
auto-activate: false
use-mamba: true
- name: run spark-3.3 tests with scala-2.12 and python-${{ matrix.python
}}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.12 -Pintegration
-DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.3 tests with scala-2.13 and python-${{ matrix.python
}}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.3 -Pspark-scala-2.13 -Pintegration
-DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.4 tests with scala-2.13 and python-${{ matrix.python
}}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.4 -Pspark-scala-2.13 -Pintegration
-DfailIfNoTests=false ${MAVEN_ARGS}
- name: run spark-3.5 tests with scala-2.13 and python-${{ matrix.python
}}
- if: ${{ matrix.java == 11 }}
run: |
rm -rf spark/interpreter/metastore_db
./mvnw verify -pl spark-submit,spark/interpreter -am
-Dtest=org/apache/zeppelin/spark/* -Pspark-3.5 -Pspark-scala-2.13 -Pintegration
-DfailIfNoTests=false ${MAVEN_ARGS}
@@ -442,7 +440,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 11, 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
@@ -472,7 +470,7 @@ jobs:
strategy:
fail-fast: false
matrix:
- java: [ 11 ]
+ java: [ 17 ]
steps:
- name: Checkout
uses: actions/checkout@v5
diff --git a/bin/common.cmd b/bin/common.cmd
index 0ddbad8a51..4931a08180 100644
--- a/bin/common.cmd
+++ b/bin/common.cmd
@@ -71,14 +71,38 @@ if not defined ZEPPELIN_JAVA_OPTS (
set ZEPPELIN_JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
-Dfile.encoding=%ZEPPELIN_ENCODING% %ZEPPELIN_MEM%
)
+REM JPMS (Java Platform Module System) args mirrored from pom.xml
extraJavaTestArgs.
+REM Targets JDK 17+: --add-modules, --enable-native-access,
--sun-misc-unsafe-memory-access
+REM require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown
-XX flags.
+set JPMS_JAVA_OPTS=-XX:+IgnoreUnrecognizedVMOptions
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.lang=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.io=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.net=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.nio=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --add-opens=java.base/java.util=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/sun.security.action=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
--add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.tryReflectionSetAccessible=true
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% -Dio.netty.allocator.type=pooled
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS%
-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
+set JPMS_JAVA_OPTS=%JPMS_JAVA_OPTS% --sun-misc-unsafe-memory-access=allow
--enable-native-access=ALL-UNNAMED
+
if not defined JAVA_OPTS (
set JAVA_OPTS=%ZEPPELIN_JAVA_OPTS%
) else (
set JAVA_OPTS=%JAVA_OPTS% %ZEPPELIN_JAVA_OPTS%
)
+set JAVA_OPTS=%JAVA_OPTS% %JPMS_JAVA_OPTS%
-set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS%
-Dfile.encoding=%ZEPPELIN_ENCODING%
+set JAVA_INTP_OPTS=%ZEPPELIN_INTP_JAVA_OPTS%
-Dfile.encoding=%ZEPPELIN_ENCODING% %JPMS_JAVA_OPTS%
if not defined JAVA_HOME (
set ZEPPELIN_RUNNER=java
diff --git a/bin/common.sh b/bin/common.sh
index 31ff76c8a6..397115a674 100644
--- a/bin/common.sh
+++ b/bin/common.sh
@@ -147,7 +147,29 @@ if [[ ( -z "${ZEPPELIN_INTP_MEM}" ) && (
"${ZEPPELIN_INTERPRETER_LAUNCHER}" != "
export ZEPPELIN_INTP_MEM="-Xmx1024m"
fi
-JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}
${ZEPPELIN_MEM}"
+# JPMS (Java Platform Module System) args mirrored from pom.xml
extraJavaTestArgs.
+# Targets JDK 17+: --add-modules, --enable-native-access,
--sun-misc-unsafe-memory-access
+# require JDK 17+. -XX:+IgnoreUnrecognizedVMOptions only silences unknown -XX
flags.
+JPMS_JAVA_OPTS="-XX:+IgnoreUnrecognizedVMOptions"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.invoke=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.lang.reflect=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.io=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.net=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.nio=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/java.util.concurrent=ALL-UNNAMED"
+JPMS_JAVA_OPTS+="
--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.ch=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.nio.cs=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.security.action=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" --add-opens=java.base/sun.util.calendar=ALL-UNNAMED"
+JPMS_JAVA_OPTS+=" -Dio.netty.tryReflectionSetAccessible=true"
+JPMS_JAVA_OPTS+=" -Dio.netty.allocator.type=pooled"
+JPMS_JAVA_OPTS+="
-Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE"
+JPMS_JAVA_OPTS+=" --sun-misc-unsafe-memory-access=allow
--enable-native-access=ALL-UNNAMED"
+JAVA_OPTS+=" ${ZEPPELIN_JAVA_OPTS} -Dfile.encoding=${ZEPPELIN_ENCODING}
${ZEPPELIN_MEM} ${JPMS_JAVA_OPTS}"
if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then
JAVA_OPTS+="
-Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties"
else
@@ -155,7 +177,7 @@ else
fi
export JAVA_OPTS
-JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS}
-Dfile.encoding=${ZEPPELIN_ENCODING}"
+JAVA_INTP_OPTS="${ZEPPELIN_INTP_JAVA_OPTS}
-Dfile.encoding=${ZEPPELIN_ENCODING} ${JPMS_JAVA_OPTS}"
if [[ -n "${ZEPPELIN_IN_DOCKER}" ]]; then
JAVA_INTP_OPTS+="
-Dlog4j.configuration=file://${ZEPPELIN_CONF_DIR}/log4j_docker.properties
-Dlog4j.configurationFile=file://${ZEPPELIN_CONF_DIR}/log4j2_docker.properties"
elif [[ -z "${ZEPPELIN_SPARK_YARN_CLUSTER}" ]]; then
diff --git a/flink/flink-scala-2.12/pom.xml b/flink/flink-scala-2.12/pom.xml
index 7d551123dd..cde7bd4591 100644
--- a/flink/flink-scala-2.12/pom.xml
+++ b/flink/flink-scala-2.12/pom.xml
@@ -34,7 +34,6 @@
<properties>
<!--library versions-->
<flink.version>${flink1.19.version}</flink.version>
- <flink.scala.version>2.12.7</flink.scala.version>
<flink.scala.binary.version>2.12</flink.scala.binary.version>
<flink.scala.compile.version>${flink.scala.version}</flink.scala.compile.version>
<hive.version>2.3.7</hive.version>
@@ -1060,7 +1059,7 @@
<!-- set sun.zip.disableMemoryMapping=true because of
https://blogs.oracle.com/poonam/crashes-in-zipgetentry
https://bugs.java.com/bugdatabase/view_bug.do?bug_id=8191484 -->
- <argLine>-Xmx5120m -XX:MaxMetaspaceSize=1024m
-Dsun.zip.disableMemoryMapping=true</argLine>
+ <argLine>-Xmx5120m -XX:MaxMetaspaceSize=1024m
-Dsun.zip.disableMemoryMapping=true ${extraJavaTestArgs}</argLine>
<!-- <argLine>-Xmx4096m -XX:MaxMetaspaceSize=512m
-Dsun.zip.disableMemoryMapping=true
-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=6006</argLine>-->
<environmentVariables>
@@ -1222,6 +1221,38 @@
<scope>provided</scope>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <!-- Flink 1.x tgz bundles flink-scala_2.12 which shades scala
2.12.7.
+ 2.12.7's JrtClassPath.asURLs() crashes on JDK 17+
(scala/bug#11608, fixed in 2.12.9).
+ Strip the shaded classes and drop in 2.12.20 jars. Not needed
for Flink 2.0+. -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>patch-flink-scala</id>
+ <phase>initialize</phase>
+ <goals><goal>run</goal></goals>
+ <configuration>
+ <target>
+ <exec executable="zip"
dir="${project.build.directory}/flink-${flink.version}/lib" failonerror="true">
+ <arg line="-d
flink-scala_${flink.scala.binary.version}-${flink.version}.jar scala/*"/>
+ </exec>
+ <copy
todir="${project.build.directory}/flink-${flink.version}/lib" flatten="true">
+ <resources>
+ <file
file="${settings.localRepository}/org/scala-lang/scala-library/${scala.2.12.version}/scala-library-${scala.2.12.version}.jar"/>
+ <file
file="${settings.localRepository}/org/scala-lang/scala-compiler/${scala.2.12.version}/scala-compiler-${scala.2.12.version}.jar"/>
+ <file
file="${settings.localRepository}/org/scala-lang/scala-reflect/${scala.2.12.version}/scala-reflect-${scala.2.12.version}.jar"/>
+ </resources>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</profile>
<profile>
@@ -1256,6 +1287,35 @@
<scope>provided</scope>
</dependency>
</dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>patch-flink-scala</id>
+ <phase>initialize</phase>
+ <goals><goal>run</goal></goals>
+ <configuration>
+ <target>
+ <exec executable="zip"
dir="${project.build.directory}/flink-${flink.version}/lib" failonerror="true">
+ <arg line="-d
flink-scala_${flink.scala.binary.version}-${flink.version}.jar scala/*"/>
+ </exec>
+ <copy
todir="${project.build.directory}/flink-${flink.version}/lib" flatten="true">
+ <resources>
+ <file
file="${settings.localRepository}/org/scala-lang/scala-library/${scala.2.12.version}/scala-library-${scala.2.12.version}.jar"/>
+ <file
file="${settings.localRepository}/org/scala-lang/scala-compiler/${scala.2.12.version}/scala-compiler-${scala.2.12.version}.jar"/>
+ <file
file="${settings.localRepository}/org/scala-lang/scala-reflect/${scala.2.12.version}/scala-reflect-${scala.2.12.version}.jar"/>
+ </resources>
+ </copy>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
</profile>
</profiles>
</project>
diff --git a/flink/pom.xml b/flink/pom.xml
index 490ee787ba..19f8ffd151 100644
--- a/flink/pom.xml
+++ b/flink/pom.xml
@@ -44,7 +44,7 @@
<flink1.19.version>1.19.3</flink1.19.version>
<flink1.20.version>1.20.4</flink1.20.version>
- <flink.scala.version>2.12.7</flink.scala.version>
+ <flink.scala.version>2.12.20</flink.scala.version>
<flink.scala.binary.version>2.12</flink.scala.binary.version>
</properties>
</project>
diff --git a/pom.xml b/pom.xml
index a8e8a7f8f8..19b13ba476 100644
--- a/pom.xml
+++ b/pom.xml
@@ -178,6 +178,29 @@
<testcontainers.version>1.21.4</testcontainers.version>
<MaxMetaspace>512m</MaxMetaspace>
+ <extraJavaTestArgs>
+ -XX:+IgnoreUnrecognizedVMOptions
+ --add-opens=java.base/java.lang=ALL-UNNAMED
+ --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
+ --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
+ --add-opens=java.base/java.io=ALL-UNNAMED
+ --add-opens=java.base/java.net=ALL-UNNAMED
+ --add-opens=java.base/java.nio=ALL-UNNAMED
+ --add-opens=java.base/java.util=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
+ --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
+ --add-opens=java.base/sun.security.action=ALL-UNNAMED
+ --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
+ -Dio.netty.tryReflectionSetAccessible=true
+ -Dio.netty.allocator.type=pooled
+ -Dio.netty.handler.ssl.defaultEndpointVerificationAlgorithm=NONE
+ --sun-misc-unsafe-memory-access=allow
+ --enable-native-access=ALL-UNNAMED
+ -XX:+EnableDynamicAgentLoading
+ </extraJavaTestArgs>
<!-- to be able to exclude some tests using command line -->
<tests.to.exclude/>
@@ -658,7 +681,7 @@
<configuration combine.children="append">
<failIfNoTests>false</failIfNoTests>
<failIfNoSpecifiedTests>false</failIfNoSpecifiedTests>
- <argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8
${extraJavaTestArgs}</argLine>
<environmentVariables>
<IS_ZEPPELIN_TEST>true</IS_ZEPPELIN_TEST>
</environmentVariables>
diff --git a/spark/interpreter/pom.xml b/spark/interpreter/pom.xml
index 0e276a49ba..a51f045ae8 100644
--- a/spark/interpreter/pom.xml
+++ b/spark/interpreter/pom.xml
@@ -55,8 +55,6 @@
<!-- settings -->
<pyspark.test.exclude>**/PySparkInterpreterMatplotlibTest.java</pyspark.test.exclude>
<pyspark.test.include>**/*Test.*</pyspark.test.include>
-
- <extraJavaTestArgs></extraJavaTestArgs>
</properties>
<dependencies>
@@ -453,36 +451,6 @@
</build>
<profiles>
-
- <profile>
- <id>java-17</id>
- <activation>
- <jdk>[17,)</jdk>
- </activation>
- <properties>
- <extraJavaTestArgs>
- -XX:+IgnoreUnrecognizedVMOptions
- --add-modules=jdk.incubator.vector
- --add-opens=java.base/java.lang=ALL-UNNAMED
- --add-opens=java.base/java.lang.invoke=ALL-UNNAMED
- --add-opens=java.base/java.lang.reflect=ALL-UNNAMED
- --add-opens=java.base/java.io=ALL-UNNAMED
- --add-opens=java.base/java.net=ALL-UNNAMED
- --add-opens=java.base/java.nio=ALL-UNNAMED
- --add-opens=java.base/java.util=ALL-UNNAMED
- --add-opens=java.base/java.util.concurrent=ALL-UNNAMED
- --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
- --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED
- --add-opens=java.base/sun.nio.ch=ALL-UNNAMED
- --add-opens=java.base/sun.nio.cs=ALL-UNNAMED
- --add-opens=java.base/sun.security.action=ALL-UNNAMED
- --add-opens=java.base/sun.util.calendar=ALL-UNNAMED
- -Djdk.reflect.useDirectMethodHandle=false
- -Dio.netty.tryReflectionSetAccessible=true
- </extraJavaTestArgs>
- </properties>
- </profile>
-
<!-- profile spark-scala-x only affect the unit test in spark/interpreter
module -->
<profile>
diff --git a/zeppelin-integration/pom.xml b/zeppelin-integration/pom.xml
index fe665f2d01..3368f7d072 100644
--- a/zeppelin-integration/pom.xml
+++ b/zeppelin-integration/pom.xml
@@ -165,7 +165,7 @@
<plugin>
<artifactId>maven-surefire-plugin</artifactId>
<configuration combine.children="append">
- <argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xmx2g -Xms1g -Dfile.encoding=UTF-8
${extraJavaTestArgs}</argLine>
<excludes>
<exclude>${tests.to.exclude}</exclude>
</excludes>
diff --git a/zeppelin-interpreter-integration/pom.xml
b/zeppelin-interpreter-integration/pom.xml
index a0a4e9ecd3..3197de27da 100644
--- a/zeppelin-interpreter-integration/pom.xml
+++ b/zeppelin-interpreter-integration/pom.xml
@@ -174,7 +174,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx3072m</argLine>
+ <argLine>-Xmx3072m ${extraJavaTestArgs}</argLine>
<environmentVariables>
<ZEPPELIN_HOME>${basedir}/../</ZEPPELIN_HOME>
</environmentVariables>
diff --git a/zeppelin-server/pom.xml b/zeppelin-server/pom.xml
index 8dbdb57567..505c5ed75b 100644
--- a/zeppelin-server/pom.xml
+++ b/zeppelin-server/pom.xml
@@ -585,7 +585,7 @@
<configuration>
<forkCount>1</forkCount>
<reuseForks>false</reuseForks>
- <argLine>-Xmx3g -Xms1g -Dfile.encoding=UTF-8</argLine>
+ <argLine>-Xmx3g -Xms1g -Dfile.encoding=UTF-8
${extraJavaTestArgs}</argLine>
<systemProperties>
<java.io.tmpdir>${project.build.directory}</java.io.tmpdir>
</systemProperties>
diff --git
a/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java
b/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java
index 8a7dd6bdc4..86b48c4531 100644
--- a/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java
+++ b/zeppelin-test/src/main/java/org/apache/zeppelin/test/DownloadUtils.java
@@ -455,6 +455,7 @@ public class DownloadUtils {
if (targetFlinkHomeFolder.exists()) {
LOGGER.info("Skip to download Flink {}_{} as it is already downloaded.",
flinkVersion,
scalaVersion);
+ patchFlinkScala(targetFlinkHomeFolder, flinkVersion, scalaVersion);
return targetFlinkHomeFolder.getAbsolutePath();
}
File flinkTGZ = new File(flinkDownloadFolder,
@@ -541,9 +542,52 @@ public class DownloadUtils {
} catch (Exception e) {
throw new RuntimeException("Fail to download jar", e);
}
+ patchFlinkScala(targetFlinkHomeFolder, flinkVersion, scalaVersion);
return targetFlinkHomeFolder.getAbsolutePath();
}
+ /**
+ * Flink 1.x tgz bundles flink-scala_2.12 which shades scala 2.12.7.
+ * Scala 2.12.7's JrtClassPath.asURLs() crashes on JDK 17+ (scala/bug#11608,
fixed in 2.12.9).
+ * Strip the shaded classes and copy in Scala 2.12.20 jars.
+ */
+ private static void patchFlinkScala(File flinkHome, String flinkVersion,
String scalaVersion) {
+ if (!"2.12".equals(scalaVersion)) {
+ return;
+ }
+ if
(SemanticVersion.of(flinkVersion).equalsOrNewerThan(SemanticVersion.of("2.0.0")))
{
+ return;
+ }
+ File libDir = new File(flinkHome, "lib");
+ for (File jar : libDir.listFiles((d, n) -> n.startsWith("flink-scala_") &&
n.endsWith(".jar"))) {
+ LOGGER.info("Stripping Scala 2.12.7 classes from {}", jar.getName());
+ try {
+ Process p = new ProcessBuilder("zip", "-d", jar.getAbsolutePath(),
"scala/*")
+ .redirectErrorStream(true).start();
+ IOUtils.toString(p.getInputStream(), StandardCharsets.UTF_8);
+ p.waitFor();
+ } catch (Exception e) {
+ LOGGER.warn("Failed to strip Scala classes from {}", jar.getName(), e);
+ }
+ }
+ String scalaPatchVersion = "2.12.20";
+ for (String artifact : new String[]{"scala-library", "scala-compiler",
"scala-reflect"}) {
+ String jarName = artifact + "-" + scalaPatchVersion + ".jar";
+ File dest = new File(libDir, jarName);
+ if (dest.exists()) {
+ continue;
+ }
+ String url = "https://repo1.maven.org/maven2/org/scala-lang/"; + artifact
+ "/"
+ + scalaPatchVersion + "/" + jarName;
+ try {
+ LOGGER.info("Downloading {} to {}", jarName, dest);
+ download(url, 3, dest);
+ } catch (Exception e) {
+ throw new RuntimeException("Fail to download " + jarName, e);
+ }
+ }
+ }
+
private static void mvFile(String srcPath, String dstPath) throws
IOException {
Path src = Paths.get(srcPath);
Path dst = Paths.get(dstPath);
