This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c081ccd0d7 [spark] Compile the shared Spark test sources against 4.0.3
for paimon-spark-4.0 (#9794)
c081ccd0d7 is described below
commit c081ccd0d7003ceae2517c6b592a947e824acc25
Author: YangJie <[email protected]>
AuthorDate: Sun Sep 13 22:57:58 2026 -0400
[spark] Compile the shared Spark test sources against 4.0.3 for
paimon-spark-4.0 (#9794)
---
.github/workflows/publish-snapshot-spark4.yml | 2 +-
paimon-spark/paimon-spark-4.0/pom.xml | 6 +-
.../StreamTestCheckAnswerWithTimeoutStub.scala | 63 -------
.../pom.xml | 184 +++++++++++----------
pom.xml | 1 +
5 files changed, 103 insertions(+), 153 deletions(-)
diff --git a/.github/workflows/publish-snapshot-spark4.yml
b/.github/workflows/publish-snapshot-spark4.yml
index c1b8eabeb0..998375b300 100644
--- a/.github/workflows/publish-snapshot-spark4.yml
+++ b/.github/workflows/publish-snapshot-spark4.yml
@@ -69,6 +69,6 @@ jobs:
mvn --settings $tmp_settings -ntp clean install -Dgpg.skip
-Drat.skip -DskipTests -Papache-release,spark4,flink1 -pl
org.apache.paimon:paimon-spark-4.0_2.13,org.apache.paimon:paimon-spark-4.1_2.13
-am
# skip deploy paimon-spark-common_2.13 since they are already
deployed in publish-snapshot.yml
- mvn --settings $tmp_settings -ntp clean deploy -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark4,flink1 -pl
org.apache.paimon:paimon-spark4-common_2.13,org.apache.paimon:paimon-spark-ut_2.13,org.apache.paimon:paimon-spark-4.0_2.13,org.apache.paimon:paimon-spark-4.1_2.13
+ mvn --settings $tmp_settings -ntp clean deploy -Dgpg.skip -Drat.skip
-DskipTests -Papache-release,spark4,flink1 -pl
org.apache.paimon:paimon-spark4-common_2.13,org.apache.paimon:paimon-spark-ut_2.13,org.apache.paimon:paimon-spark-ut-4.0_2.13,org.apache.paimon:paimon-spark-4.0_2.13,org.apache.paimon:paimon-spark-4.1_2.13
rm $tmp_settings
diff --git a/paimon-spark/paimon-spark-4.0/pom.xml
b/paimon-spark/paimon-spark-4.0/pom.xml
index d84e6e3ae9..1a0f80701d 100644
--- a/paimon-spark/paimon-spark-4.0/pom.xml
+++ b/paimon-spark/paimon-spark-4.0/pom.xml
@@ -124,9 +124,13 @@ under the License.
<!-- test -->
+ <!-- The 4.0-compiled build of `paimon-spark-ut`'s test sources. The
baseline build cannot be
+ used here: it is compiled against the `spark4` profile baseline,
so its classfiles can
+ reference Spark symbols that a 4.0 runtime does not have. See
+ `paimon-spark-ut-4.0/pom.xml`. -->
<dependency>
<groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
+
<artifactId>paimon-spark-ut-4.0_${scala.binary.version}</artifactId>
<version>${project.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
diff --git
a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/spark/sql/streaming/StreamTestCheckAnswerWithTimeoutStub.scala
b/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/spark/sql/streaming/StreamTestCheckAnswerWithTimeoutStub.scala
deleted file mode 100644
index bb6edfa9ae..0000000000
---
a/paimon-spark/paimon-spark-4.0/src/test/scala/org/apache/spark/sql/streaming/StreamTestCheckAnswerWithTimeoutStub.scala
+++ /dev/null
@@ -1,63 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.streaming
-
-/**
- * Test-scope stubs for Spark 4.1-only `StreamTest` inner classes.
- *
- * Spark 4.1 added several `CheckAnswer` / `WaitUntil` helpers inside
`StreamTest`; Paimon's shared
- * test base classes in `paimon-spark-ut` mix in `StreamTest` and are compiled
against Spark 4.1.1
- * (the default `spark.version` under the `spark4` profile), so their compiled
bytecode has
- * constant-pool references to the nested companion objects.
- *
- * When those test classes are loaded on a Spark 4.0.2 runtime (the
paimon-spark-4.0 test suite),
- * `junit-vintage`'s discovery phase tries to verify each class and fails to
link the missing 4.1
- * symbols, crashing the engine before any Paimon test can run. No Paimon test
actually invokes
- * these helpers — the references only exist in the constant pool from
trait-mix-in synthetic
- * forwarders — so empty class files with the matching names are enough to
satisfy the verifier.
- *
- * Diffing `spark-sql_2.13-4.1.1-tests.jar` against
`spark-sql_2.13-4.0.1-tests.jar` shows the
- * following `StreamTest$…` classes are 4.1-only:
- *
- * companion objects (always referenced via MODULE$ access):
- * - `StreamTest$CheckAnswerWithTimeout$`
- * - `StreamTest$CheckAnswerRowsContainsWithTimeout$`
- * - `StreamTest$CheckAnswerRowsNoWait$`
- * - `StreamTest$WaitUntilBatchProcessed$`
- * - `StreamTest$WaitUntilCurrentBatchProcessed$`
- *
- * case classes (may be referenced as parameter / return types):
- * - `StreamTest$CheckAnswerRowsContainsWithTimeout`
- * - `StreamTest$CheckAnswerRowsNoWait`
- * - `StreamTest$WaitUntilBatchProcessed`
- *
- * Scala's backtick syntax lets us declare classes whose compiled names
contain `$`, so each stub
- * compiles to exactly the file name expected by the verifier. They live in
`src/test/scala/` so
- * they are never packaged into any production jar (including the
paimon-spark-4.0 shaded artifact)
- * and do not leak onto Spark 4.1 classpaths where the real symbols must win.
- */
-private[streaming] class `StreamTest$CheckAnswerWithTimeout$`
-private[streaming] class `StreamTest$CheckAnswerRowsContainsWithTimeout$`
-private[streaming] class `StreamTest$CheckAnswerRowsNoWait$`
-private[streaming] class `StreamTest$WaitUntilBatchProcessed$`
-private[streaming] class `StreamTest$WaitUntilCurrentBatchProcessed$`
-
-private[streaming] class `StreamTest$CheckAnswerRowsContainsWithTimeout`
-private[streaming] class `StreamTest$CheckAnswerRowsNoWait`
-private[streaming] class `StreamTest$WaitUntilBatchProcessed`
diff --git a/paimon-spark/paimon-spark-4.0/pom.xml
b/paimon-spark/paimon-spark-ut-4.0/pom.xml
similarity index 52%
copy from paimon-spark/paimon-spark-4.0/pom.xml
copy to paimon-spark/paimon-spark-ut-4.0/pom.xml
index d84e6e3ae9..b4a96db574 100644
--- a/paimon-spark/paimon-spark-4.0/pom.xml
+++ b/paimon-spark/paimon-spark-ut-4.0/pom.xml
@@ -28,23 +28,49 @@ under the License.
<version>2.2-SNAPSHOT</version>
</parent>
- <artifactId>paimon-spark-4.0_2.13</artifactId>
- <name>Paimon : Spark : 4.0 : 2.13</name>
-
+ <artifactId>paimon-spark-ut-4.0_${scala.binary.version}</artifactId>
+ <name>Paimon : Spark : UT : 4.0 : ${scala.binary.version}</name>
+
+ <!--
+ A second compilation of `paimon-spark-ut`'s test sources, against Spark
4.0.3 instead of the
+ `spark4` profile baseline. Owns no sources of its own: `build-helper`
adds
+ `../paimon-spark-ut/src/test/scala` and `src/test/java` as test-source
roots, and
+ `<testResources>` points at that module's test resources, so there is
exactly one copy of every
+ test to maintain.
+
+ Why a whole module rather than a second classifier on `paimon-spark-ut`:
a Maven module has one
+ dependency set, so producing two artifacts compiled against different
Spark versions from one
+ module is not expressible.
+
+ Why it is needed at all: `paimon-spark-4.0` used to consume
`paimon-spark-ut`'s test-jar, which
+ is compiled against the baseline. Those classfiles reference Spark
symbols that a 4.0 runtime
+ cannot resolve, which is what `StreamTestCheckAnswerWithTimeoutStub`
stood in for. Compiling
+ the same sources against 4.0.3 cannot emit a reference to a symbol 4.0.3
does not have, so the
+ stub is deleted here.
+
+ Serves `paimon-spark-4.0` only: `paimon-spark-4.1` pins 4.1.2, which is
the baseline, so the
+ baseline build already matches its runtime and it keeps consuming
`paimon-spark-ut` directly.
+ -->
<properties>
<spark.version>4.0.3</spark.version>
- <!-- Spark 4.x uses SLF4J 2.x (`org.slf4j.spi.LoggingEventBuilder` is
2.x-only). Override
- the Paimon parent pom's default `slf4j.version=1.7.32` so the
classpath is uniformly
- 2.x; otherwise `log4j-slf4j2-impl` + `slf4j-api:1.7.32` mix
produces
- `NoClassDefFoundError: org/slf4j/spi/LoggingEventBuilder` at test
startup. -->
- <slf4j.version>2.0.16</slf4j.version>
+ <ut.module.dir>${project.basedir}/../paimon-spark-ut</ut.module.dir>
<!-- Spark 4 only supports Scala 2.13. Pin it locally so this _2.13
module resolves its
dependencies to _2.13 even when the spark4 profile is inactive
(see issue #6682);
otherwise it inherits scala.binary.version=2.12 from
paimon-parent. -->
<scala.binary.version>2.13</scala.binary.version>
+ <!-- The tests belong to `paimon-spark-ut`, which runs them against
the baseline. This module
+ exists only to produce a 4.0-compiled test-jar, so nothing should
run them a second time
+ here. Both surefire and scalatest-maven-plugin bind their
`skipTests` parameter to this
+ property; `maven.test.skip` would also skip test compilation and
leave the test-jar
+ empty. Without this, surefire inherits the root pom's
`default-test` and
+ `integration-tests` executions and fails on `TestLoggerExtension`
with
+ `NoClassDefFoundError: org/slf4j/spi/LoggingEventBuilder`. -->
+ <skipTests>true</skipTests>
</properties>
<dependencies>
+ <!-- Add to the beginning, just to make IDEA prioritize loading
Paimon's conflicting classes,
+ such as Parquet and ORC. Remove this once IDEA's bug is fixed. -->
<dependency>
<groupId>org.apache.paimon</groupId>
<artifactId>paimon-format</artifactId>
@@ -52,31 +78,24 @@ under the License.
<dependency>
<groupId>org.apache.paimon</groupId>
-
<artifactId>paimon-spark4-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
-
<artifactId>paimon-spark-common_${scala.binary.version}</artifactId>
+ <artifactId>${paimon-sparkx-common}</artifactId>
<version>${project.version}</version>
</dependency>
<!--
- `paimon-spark4-common` transitively pulls in `spark-sql-api` at the
version set by
- `paimon-spark-common.spark.version` (4.1.2 under the `spark4`
profile). Spark 4.1's
- `SqlApiConf` added abstract `manageParserCaches()`, which Spark
4.0's concrete
- anonymous subclasses (e.g. `WithTestConf$$anon$4`) don't implement.
Mixing those
- jars produces `AbstractMethodError` at parser configuration time. Pin
- `spark-sql-api` to `${spark.version}` (4.0.3) so the test classpath
is uniformly 4.0.
+ `${paimon-sparkx-common}` transitively pulls in `spark-sql-api` at
the `spark4` profile
+ baseline (4.1.2), so every Spark artifact is pinned to
${spark.version} here to keep the
+ whole compile classpath on 4.0. Same reasoning as
`paimon-spark-4.0`, and the same
+ `spark-connect-shims` exclusion on `spark-sql-api` and `spark-sql`:
Spark 4.0.3's shims jar
+ carries a stub `org.apache.spark.SparkConf` with only `getAll()`,
which scalac resolves in
+ preference to spark-core's real one, failing with `value set is not
a member of
+ org.apache.spark.SparkConf`.
-->
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql-api_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
- <!-- See spark-sql exclusion below: spark-connect-shims ships
a stub SparkConf
- that shadows the real one from spark-core. -->
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-connect-shims_${scala.binary.version}</artifactId>
@@ -89,14 +108,6 @@ under the License.
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<exclusions>
- <!--
- Spark 4.0.3 ships a tiny `spark-connect-shims` jar
containing a stub
- `org.apache.spark.SparkConf` with only `getAll()` and a
no-arg constructor
- (no `set` etc.). scalac resolves that stub in preference to
spark-core's
- real `SparkConf` and fails compile with `value set is not a
member of
- org.apache.spark.SparkConf`. The shims jar is only relevant
to Spark Connect
- clients; excluding it leaves the full `SparkConf` from
`spark-core`.
- -->
<exclusion>
<groupId>org.apache.spark</groupId>
<artifactId>spark-connect-shims_${scala.binary.version}</artifactId>
@@ -104,12 +115,6 @@ under the License.
</exclusions>
</dependency>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- </dependency>
-
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-catalyst_${scala.binary.version}</artifactId>
@@ -118,103 +123,106 @@ under the License.
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_${scala.binary.version}</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
</dependency>
- <!-- test -->
+ <!-- Below are test dependencies which need to be distinguished for
different Spark versions. -->
<dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-spark-ut_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
- <groupId>org.apache.paimon</groupId>
-
<artifactId>paimon-spark4-common_${scala.binary.version}</artifactId>
- <version>${project.version}</version>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>*</groupId>
- <artifactId>*</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<classifier>tests</classifier>
<scope>test</scope>
- <exclusions>
- <exclusion>
- <groupId>org.apache.spark</groupId>
-
<artifactId>spark-connect-shims_${scala.binary.version}</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-catalyst_${scala.binary.version}</artifactId>
+ <artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
- <classifier>tests</classifier>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <artifactId>spark-avro_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
- <classifier>tests</classifier>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-lumina</artifactId>
+ <version>${project.version}</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
+ <testResources>
+ <testResource>
+ <directory>${ut.module.dir}/src/test/resources</directory>
+ </testResource>
+ </testResources>
+
<plugins>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
<executions>
<execution>
- <id>shade-paimon</id>
- <phase>package</phase>
+ <id>add-ut-test-sources</id>
+ <phase>generate-test-sources</phase>
<goals>
- <goal>shade</goal>
+ <goal>add-test-source</goal>
</goals>
<configuration>
- <filters>
- <filter>
- <artifact>*</artifact>
- <excludes>
-
<exclude>com/github/luben/zstd/**</exclude>
- <exclude>**/*libzstd-jni-*.so</exclude>
-
<exclude>**/*libzstd-jni-*.dll</exclude>
- </excludes>
- </filter>
- </filters>
- <artifactSet>
- <includes combine.children="append">
-
<include>org.apache.paimon:paimon-spark4-common_${scala.binary.version}</include>
- </includes>
- </artifactSet>
+ <sources>
+
<source>${ut.module.dir}/src/test/scala</source>
+ <source>${ut.module.dir}/src/test/java</source>
+ </sources>
</configuration>
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-jar-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>prepare-test-jar</id>
+ <phase>test-compile</phase>
+ <goals>
+ <goal>test-jar</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/pom.xml b/pom.xml
index f4d2c5b1da..0b5512bd31 100644
--- a/pom.xml
+++ b/pom.xml
@@ -429,6 +429,7 @@ under the License.
<id>spark4</id>
<modules>
<module>paimon-spark/paimon-spark4-common</module>
+ <module>paimon-spark/paimon-spark-ut-4.0</module>
<module>paimon-spark/paimon-spark-4.0</module>
<module>paimon-spark/paimon-spark-4.1</module>
</modules>
