This is an automated email from the ASF dual-hosted git repository.
Gabriel39 pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 776adf28a07 branch-4.1: [chore](paimon) Upgrade Paimon to 1.4.2
(#66346)
776adf28a07 is described below
commit 776adf28a07224de1b89207f81e829dcff98daf7
Author: Socrates <[email protected]>
AuthorDate: Wed Aug 5 23:44:47 2026 +0800
branch-4.1: [chore](paimon) Upgrade Paimon to 1.4.2 (#66346)
## What changes
- Upgrade the FE Paimon dependency from 1.3.1 to 1.4.2.
- Update the Paimon test IO manager implementation for the new
`pickTempDir()` API.
## Why
Use the latest stable Paimon release and keep the Doris Paimon connector
compatible with its updated IO manager interface.
## Validation
- `DISABLE_BUILD_UI=ON ./build.sh --fe`
- FE Maven reactor completed successfully with `BUILD SUCCESS`.
- UI compilation was skipped.
- The build reused the main checkout's `thirdparty/installed`.
---
.../docker-compose/iceberg/iceberg.yaml.tpl | 4 +-
docker/thirdparties/run-thirdparties-docker.sh | 6 +-
.../java-udf/src/main/resources/package.xml | 8 ++
fe/be-java-extensions/paimon-connector/pom.xml | 11 +++
.../org/apache/doris/paimon/PaimonJniScanner.java | 18 +++--
.../apache/doris/paimon/PaimonJniScannerTest.java | 57 +++++++++-----
fe/be-java-extensions/preload-extensions/pom.xml | 12 +--
fe/fe-core/pom.xml | 86 ++++++++++++++++------
.../datasource/paimon/PaimonReaderOptions.java | 18 +++--
.../apache/doris/datasource/paimon/PaimonUtil.java | 3 +-
.../datasource/paimon/source/PaimonScanNode.java | 12 ++-
.../datasource/paimon/PaimonReaderOptionsTest.java | 41 +++++++----
.../doris/datasource/paimon/PaimonUtilTest.java | 2 +-
.../paimon/source/PaimonScanNodeTest.java | 20 ++++-
.../PaimonTableValuedFunctionTest.java | 2 +-
fe/pom.xml | 2 +-
.../paimon/paimon_system_table.out | 11 +--
.../paimon/test_paimon_partition_table.out | 38 +++++-----
.../paimon_write/test_paimon_write_append_only.out | 10 +--
.../test_paimon_write_schema_change.out | 28 +++----
.../paimon/paimon_system_table.groovy | 3 +-
21 files changed, 254 insertions(+), 138 deletions(-)
diff --git a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
index d62a3eee361..5fc6af63e77 100644
--- a/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
+++ b/docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl
@@ -36,8 +36,8 @@ services:
-
./data/input/jars/iceberg-aws-bundle-1.10.1.jar:/opt/spark/jars/iceberg-aws-bundle-1.10.1.jar
-
./data/input/jars/iceberg-spark-runtime-4.0_2.13-1.10.1.jar:/opt/spark/jars/iceberg-spark-runtime-4.0_2.13-1.10.1.jar
-
./data/input/jars/lance-spark-bundle-4.0_2.13-0.4.0.jar:/opt/spark/jars/lance-spark-bundle-4.0_2.13-0.4.0.jar
- -
./data/input/jars/paimon-s3-1.3.1.jar:/opt/spark/jars/paimon-s3-1.3.1.jar
- -
./data/input/jars/paimon-spark-4.0-1.3.1.jar:/opt/spark/jars/paimon-spark-4.0-1.3.1.jar
+ -
./data/input/jars/paimon-s3-1.4.2.jar:/opt/spark/jars/paimon-s3-1.4.2.jar
+ -
./data/input/jars/paimon-spark-4.0_2.13-1.4.2.jar:/opt/spark/jars/paimon-spark-4.0_2.13-1.4.2.jar
environment:
- AWS_ACCESS_KEY_ID=admin
- AWS_SECRET_ACCESS_KEY=password
diff --git a/docker/thirdparties/run-thirdparties-docker.sh
b/docker/thirdparties/run-thirdparties-docker.sh
index a49006f5fa1..63154a15473 100755
--- a/docker/thirdparties/run-thirdparties-docker.sh
+++ b/docker/thirdparties/run-thirdparties-docker.sh
@@ -1405,10 +1405,10 @@ start_iceberg() {
(
cd "${ICEBERG_DIR}" || exit 1
rm -f iceberg_data*.zip
- wget -P "${ROOT}/docker-compose/iceberg"
"https://${s3BucketName}.${s3Endpoint}/regression/datalake/pipeline_data/iceberg_data_spark40.zip";
- sudo unzip iceberg_data_spark40.zip
+ wget -P "${ROOT}/docker-compose/iceberg"
"https://${s3BucketName}.${s3Endpoint}/regression/datalake/pipeline_data/iceberg_data_spark40_paimon142.zip";
+ sudo unzip iceberg_data_spark40_paimon142.zip
sudo mv iceberg_data data
- sudo rm -rf iceberg_data_spark40.zip
+ sudo rm -rf iceberg_data_spark40_paimon142.zip
)
else
echo "${ICEBERG_DIR}/data exists, continue !"
diff --git a/fe/be-java-extensions/java-udf/src/main/resources/package.xml
b/fe/be-java-extensions/java-udf/src/main/resources/package.xml
index 4bbb2610603..9ef59a2ee2d 100644
--- a/fe/be-java-extensions/java-udf/src/main/resources/package.xml
+++ b/fe/be-java-extensions/java-udf/src/main/resources/package.xml
@@ -34,6 +34,14 @@ under the License.
<unpackOptions>
<excludes>
<exclude>**/Log4j2Plugins.dat</exclude>
+ <!-- hive-catalog-shade embeds Paimon 1.3 classes. This
fat jar is on the
+ JVM application classpath and therefore wins
parent-first resolution
+ over paimon-connector's isolated Paimon runtime. Java
UDF does not use
+ Paimon; keep those embedded classes out so the
connector owns its SDK. -->
+ <exclude>org/apache/paimon/**</exclude>
+ <!-- ServiceLoader also consults parent resources. Do not
retain service
+ descriptors whose providers were removed with the
embedded SDK. -->
+ <exclude>META-INF/services/org.apache.paimon*</exclude>
</excludes>
</unpackOptions>
</dependencySet>
diff --git a/fe/be-java-extensions/paimon-connector/pom.xml
b/fe/be-java-extensions/paimon-connector/pom.xml
index 4ca2f561312..51e5c682bce 100644
--- a/fe/be-java-extensions/paimon-connector/pom.xml
+++ b/fe/be-java-extensions/paimon-connector/pom.xml
@@ -61,6 +61,17 @@ under the License.
<artifactId>paimon-format</artifactId>
</dependency>
+ <!-- Keep FileIO service providers with the paimon-common SPI in this
connector's
+ isolated classloader. Loading them from preload-extensions breaks
provider linkage. -->
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-jindo</artifactId>
+ </dependency>
+
<dependency>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-vector</artifactId>
diff --git
a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
index 49c77314d4c..c7731dcdc38 100644
---
a/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
+++
b/fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java
@@ -714,14 +714,15 @@ public class PaimonJniScanner extends JniScanner {
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable)
table;
FileStoreTable main = applyManifestParallelismBound(
pair.wrapped(), safeBound, materializeAbsent);
- FileStoreTable fallback = applyManifestParallelismBound(
- pair.fallback(), safeBound, materializeAbsent);
- if (main == pair.wrapped() && fallback == pair.fallback()) {
+ FileStoreTable other = applyManifestParallelismBound(
+ pair.other(), safeBound, materializeAbsent);
+ if (main == pair.wrapped() && other == pair.other()) {
return table;
}
// Each branch owns an independent planner setting; a smaller
sibling is not an
// execution ceiling and must never throttle the other branch.
- return new FallbackReadFileStoreTable(main, fallback);
+ return new FallbackReadFileStoreTable(
+ main, other, wrappedBranchHasReadPriority(pair));
}
if (table instanceof DelegatedFileStoreTable) {
@@ -784,6 +785,11 @@ public class PaimonJniScanner extends JniScanner {
(Table) table, safeBound, materializeAbsent);
}
+ private static boolean
wrappedBranchHasReadPriority(FallbackReadFileStoreTable table) {
+ // Paimon's factory sets wrappedFirst=false only for
scan.primary-branch tables.
+ return
!table.wrapped().options().containsKey(CoreOptions.SCAN_PRIMARY_BRANCH.key());
+ }
+
private static FileStoreTable unwrapSystemPlanningSource(FileStoreTable
table) {
FileStoreTable current = table;
// System wrappers dispatch fallback reads only when the fallback pair
is their direct
@@ -812,7 +818,7 @@ public class PaimonJniScanner extends JniScanner {
return true;
}
if (table instanceof FallbackReadFileStoreTable
- && hasReadBatchSize(((FallbackReadFileStoreTable)
table).fallback())) {
+ && hasReadBatchSize(((FallbackReadFileStoreTable)
table).other())) {
return true;
}
return table instanceof DelegatedFileStoreTable
@@ -824,7 +830,7 @@ public class PaimonJniScanner extends JniScanner {
validateSerializedAsyncThreshold(table.options().get(CoreOptions.FILE_READER_ASYNC_THRESHOLD.key()));
validateSerializedSplitTargetSize(table.options().get(CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key()));
if (table instanceof FallbackReadFileStoreTable) {
- validateSerializedReaderOptions(((FallbackReadFileStoreTable)
table).fallback());
+ validateSerializedReaderOptions(((FallbackReadFileStoreTable)
table).other());
}
if (table instanceof DelegatedFileStoreTable) {
validateSerializedReaderOptions(((DelegatedFileStoreTable)
table).wrapped());
diff --git
a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java
b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java
index 8d040bfc40d..8c1209a1387 100644
---
a/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java
+++
b/fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java
@@ -138,7 +138,8 @@ public class PaimonJniScannerTest {
FileStoreTable main =
serializableFileStoreTable(Collections.emptyMap());
FileStoreTable fallback =
serializableFileStoreTable(Collections.singletonMap(
CoreOptions.FILE_READER_ASYNC_THRESHOLD.key(), "2 GB"));
- for (Table configuredTable : Arrays.asList(visible, new
FallbackReadFileStoreTable(main, fallback))) {
+ for (Table configuredTable : Arrays.asList(
+ visible, new FallbackReadFileStoreTable(main, fallback,
true))) {
Map<String, String> params = createBaseParams();
params.put("serialized_table",
Base64.getUrlEncoder().withoutPadding().encodeToString(
InstantiationUtil.serializeObject(configuredTable)));
@@ -160,7 +161,7 @@ public class PaimonJniScannerTest {
FileStoreTable fallback =
serializableFileStoreTable(Collections.singletonMap(
CoreOptions.SOURCE_SPLIT_TARGET_SIZE.key(), "0 B"));
Table filesTable = SystemTableLoader.load(
- "files", new FallbackReadFileStoreTable(main, fallback));
+ "files", new FallbackReadFileStoreTable(main, fallback, true));
IllegalArgumentException failure = Assert.assertThrows(
IllegalArgumentException.class,
@@ -175,7 +176,7 @@ public class PaimonJniScannerTest {
FileStoreTable fallback =
serializableFileStoreTable(Collections.singletonMap(
CoreOptions.READ_BATCH_SIZE.key(), "0"));
Table readerBackedSystemTable = SystemTableLoader.load(
- "audit_log", new FallbackReadFileStoreTable(main, fallback));
+ "audit_log", new FallbackReadFileStoreTable(main, fallback,
true));
Map<String, String> params = createBaseParams();
params.put("serialized_table",
Base64.getUrlEncoder().withoutPadding().encodeToString(
InstantiationUtil.serializeObject(readerBackedSystemTable)));
@@ -221,7 +222,7 @@ public class PaimonJniScannerTest {
try {
PaimonJniScanner.applyDefaultReadBatchSize(
- new FallbackReadFileStoreTable(main, fallback), 1024);
+ new FallbackReadFileStoreTable(main, fallback, true),
1024);
Assert.fail("a hidden old-FE batch size must not reach the
fallback reader");
} catch (IllegalArgumentException e) {
Assert.assertTrue(e.getMessage().contains(CoreOptions.READ_BATCH_SIZE.key()));
@@ -235,10 +236,10 @@ public class PaimonJniScannerTest {
Collections.singletonMap(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "8"));
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
- new FallbackReadFileStoreTable(main, fallback), "8", 4);
+ new FallbackReadFileStoreTable(main, fallback, true), "8", 4);
Assert.assertTrue(safe instanceof FallbackReadFileStoreTable);
- Assert.assertEquals("4", ((FallbackReadFileStoreTable) safe).fallback()
+ Assert.assertEquals("4", ((FallbackReadFileStoreTable) safe).other()
.options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
}
@@ -249,9 +250,9 @@ public class PaimonJniScannerTest {
Collections.singletonMap(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "200"));
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
- new FallbackReadFileStoreTable(main, fallback), "32", 64);
+ new FallbackReadFileStoreTable(main, fallback, true), "32",
64);
- Assert.assertEquals("32", ((FallbackReadFileStoreTable)
safe).fallback()
+ Assert.assertEquals("32", ((FallbackReadFileStoreTable) safe).other()
.options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
}
@@ -266,11 +267,11 @@ public class PaimonJniScannerTest {
Table safeVisible = PaimonJniScanner.applyBackendManifestParallelism(
visible, null, 512);
Table safeFallback = PaimonJniScanner.applyBackendManifestParallelism(
- new FallbackReadFileStoreTable(main, fallback), null, 512);
+ new FallbackReadFileStoreTable(main, fallback, true), null,
512);
Assert.assertEquals("256", safeVisible.options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
- Assert.assertEquals("256", ((FallbackReadFileStoreTable)
safeFallback).fallback()
+ Assert.assertEquals("256", ((FallbackReadFileStoreTable)
safeFallback).other()
.options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
}
@@ -290,23 +291,38 @@ public class PaimonJniScannerTest {
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "1"));
FileStoreTable fallback =
serializableFileStoreTable(Collections.singletonMap(
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128"));
- Table pair = new FallbackReadFileStoreTable(main, fallback);
+ Table pair = new FallbackReadFileStoreTable(main, fallback, true);
FallbackReadFileStoreTable unchanged = (FallbackReadFileStoreTable)
PaimonJniScanner.applyBackendManifestParallelism(pair, "128",
128);
Assert.assertEquals("1", unchanged.wrapped().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
- Assert.assertEquals("128", unchanged.fallback().options()
+ Assert.assertEquals("128", unchanged.other().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
FallbackReadFileStoreTable capped = (FallbackReadFileStoreTable)
PaimonJniScanner.applyBackendManifestParallelism(pair, "128",
64);
Assert.assertEquals("1", capped.wrapped().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
- Assert.assertEquals("64", capped.fallback().options()
+ Assert.assertEquals("64", capped.other().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
}
+ @Test
+ public void testBackendCapPreservesPrimaryBranchPriority() throws
Exception {
+ Map<String, String> mainOptions = new HashMap<>();
+ mainOptions.put(CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary");
+ mainOptions.put(CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128");
+ FileStoreTable main = serializableFileStoreTable(mainOptions);
+ FileStoreTable primary =
serializableFileStoreTable(Collections.emptyMap());
+ Table safe = PaimonJniScanner.applyBackendManifestParallelism(
+ new FallbackReadFileStoreTable(main, primary, false), null,
64);
+
+ Field wrappedFirst =
FallbackReadFileStoreTable.class.getDeclaredField("wrappedFirst");
+ wrappedFirst.setAccessible(true);
+ Assert.assertFalse(wrappedFirst.getBoolean(safe));
+ }
+
@Test
public void testBackendCapTraversesPrivilegeDelegate() {
FileStoreTable main =
serializableFileStoreTable(Collections.singletonMap(
@@ -318,7 +334,7 @@ public class PaimonJniScannerTest {
new Class<?>[] {PrivilegeChecker.class},
(proxy, method, args) -> null);
FileStoreTable privileged = PrivilegedFileStoreTable.wrap(
- new FallbackReadFileStoreTable(main, fallback), checker,
+ new FallbackReadFileStoreTable(main, fallback, true), checker,
Identifier.create("db", "table"));
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
@@ -331,7 +347,7 @@ public class PaimonJniScannerTest {
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable)
planningTable;
Assert.assertEquals("1", pair.wrapped().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
- Assert.assertEquals("64", pair.fallback().options()
+ Assert.assertEquals("64", pair.other().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
}
@@ -342,7 +358,7 @@ public class PaimonJniScannerTest {
FileStoreTable fallback =
serializableFileStoreTable(Collections.singletonMap(
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128"));
Table wrapper = SystemTableLoader.load(
- "partitions", new FallbackReadFileStoreTable(main, fallback));
+ "partitions", new FallbackReadFileStoreTable(main, fallback,
true));
Table safe = PaimonJniScanner.applyBackendManifestParallelism(
wrapper, null, 64);
@@ -352,7 +368,7 @@ public class PaimonJniScannerTest {
Assert.assertEquals("1", pair.wrapped().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
- Assert.assertEquals("64", pair.fallback().options()
+ Assert.assertEquals("64", pair.other().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
}
@@ -367,7 +383,7 @@ public class PaimonJniScannerTest {
new Class<?>[] {PrivilegeChecker.class},
(proxy, method, args) -> null);
FileStoreTable privileged = PrivilegedFileStoreTable.wrap(
- new FallbackReadFileStoreTable(main, fallback), checker,
+ new FallbackReadFileStoreTable(main, fallback, true), checker,
Identifier.create("db", "table"));
Table wrapper = SystemTableLoader.load("partitions", privileged);
@@ -838,6 +854,11 @@ public class PaimonJniScannerTest {
return tempDirs;
}
+ @Override
+ public String pickTempDir() {
+ return tempDirs.length == 0 ? null : tempDirs[0];
+ }
+
@Override
public FileIOChannel.Enumerator createChannelEnumerator() {
throw new UnsupportedOperationException();
diff --git a/fe/be-java-extensions/preload-extensions/pom.xml
b/fe/be-java-extensions/preload-extensions/pom.xml
index ecdecf236a3..8113589e99c 100644
--- a/fe/be-java-extensions/preload-extensions/pom.xml
+++ b/fe/be-java-extensions/preload-extensions/pom.xml
@@ -78,16 +78,8 @@ under the License.
</exclusion>
</exclusions>
</dependency>
- <!-- For BE Paimon OSS/S3 Access -->
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-s3</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-jindo</artifactId>
- </dependency>
+ <!-- Paimon FileIO plugins are packaged with paimon-connector so their
service providers
+ and the paimon-common SPI are loaded by the same classloader. -->
<!-- For Avro and Hudi Scanner PreLoad -->
<dependency>
<groupId>org.apache.hadoop</groupId>
diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml
index 9b8240c9610..e129de778e7 100644
--- a/fe/fe-core/pom.xml
+++ b/fe/fe-core/pom.xml
@@ -468,6 +468,35 @@ under the License.
<groupId>com.google.flatbuffers</groupId>
<artifactId>flatbuffers-java</artifactId>
</dependency>
+ <!-- Must precede hive-catalog-shade: that published shade artifact
embeds
+ Paimon 1.3 classes, while Doris compiles against paimon.version.
-->
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-core</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-common</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-hive-connector-3.1</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-format</artifactId>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-s3</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-jindo</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.doris</groupId>
<artifactId>hive-catalog-shade</artifactId>
@@ -578,30 +607,6 @@ under the License.
<artifactId>iceberg-aws</artifactId>
<version>${iceberg.version}</version>
</dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-core</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-common</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-format</artifactId>
- </dependency>
-
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-s3</artifactId>
- </dependency>
- <dependency>
- <groupId>org.apache.paimon</groupId>
- <artifactId>paimon-jindo</artifactId>
- </dependency>
-
<dependency>
<groupId>software.amazon.awssdk</groupId>
<artifactId>glue</artifactId>
@@ -1115,11 +1120,44 @@ under the License.
<overWriteSnapshots>false</overWriteSnapshots>
<overWriteIfNewer>true</overWriteIfNewer>
<includeScope>runtime</includeScope>
+ <!-- hive-catalog-shade 3.1.1 embeds Paimon 1.3.
The FE supplies
+ Paimon from the managed paimon.version
instead. Repackage this
+ dependency below without its embedded Paimon
runtime. -->
+
<excludeArtifactIds>hive-catalog-shade</excludeArtifactIds>
<skip>${skip.plugin}</skip>
</configuration>
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <version>3.1.0</version>
+ <executions>
+ <execution>
+ <id>package-hive-catalog-shade-without-paimon</id>
+ <phase>package</phase>
+ <configuration>
+ <target>
+ <delete
dir="${project.build.directory}/hive-catalog-shade-without-paimon"/>
+ <mkdir
dir="${project.build.directory}/hive-catalog-shade-without-paimon"/>
+ <unzip
src="${settings.localRepository}/org/apache/doris/hive-catalog-shade/${doris.hive.catalog.shade.version}/hive-catalog-shade-${doris.hive.catalog.shade.version}.jar"
+
dest="${project.build.directory}/hive-catalog-shade-without-paimon">
+ <patternset>
+ <exclude name="org/apache/paimon/**"/>
+ <exclude
name="META-INF/services/org.apache.paimon*"/>
+ </patternset>
+ </unzip>
+ <jar
destfile="${project.build.directory}/lib/hive-catalog-shade-${doris.hive.catalog.shade.version}.jar"
+
basedir="${project.build.directory}/hive-catalog-shade-without-paimon"/>
+ </target>
+ </configuration>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>cobertura-maven-plugin</artifactId>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java
index dfd692fb1ca..af352f37439 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonReaderOptions.java
@@ -183,10 +183,11 @@ public final class PaimonReaderOptions {
FallbackReadFileStoreTable pair = (FallbackReadFileStoreTable)
table;
FileStoreTable main = normalizeManifestParallelism(
pair.wrapped(), safeBound, materializeAbsent);
- FileStoreTable fallback = normalizeManifestParallelism(
- pair.fallback(), safeBound, materializeAbsent);
- return main == pair.wrapped() && fallback == pair.fallback()
- ? table : new FallbackReadFileStoreTable(main, fallback);
+ FileStoreTable other = normalizeManifestParallelism(
+ pair.other(), safeBound, materializeAbsent);
+ return main == pair.wrapped() && other == pair.other()
+ ? table : new FallbackReadFileStoreTable(
+ main, other, wrappedBranchHasReadPriority(pair));
}
if (table instanceof DelegatedFileStoreTable) {
@@ -235,12 +236,17 @@ public final class PaimonReaderOptions {
(Table) table, safeBound, materializeAbsent);
}
+ private static boolean
wrappedBranchHasReadPriority(FallbackReadFileStoreTable table) {
+ // Paimon's factory sets wrappedFirst=false only for
scan.primary-branch tables.
+ return
!table.wrapped().options().containsKey(CoreOptions.SCAN_PRIMARY_BRANCH.key());
+ }
+
public static void validateEffectiveTable(Table table) {
validateEffectiveTableOptions(table.options());
if (table instanceof FallbackReadFileStoreTable) {
// The fallback scan plans its private child independently, so the
visible main options
// cannot prove that every manifest executor input is safe.
- validateEffectiveTable(((FallbackReadFileStoreTable)
table).fallback());
+ validateEffectiveTable(((FallbackReadFileStoreTable)
table).other());
}
if (table instanceof DelegatedFileStoreTable) {
// Privilege and other supported delegates can hide a fallback
planner behind their
@@ -255,7 +261,7 @@ public final class PaimonReaderOptions {
validateIfPresentForRuntime(table.options(),
CoreOptions.SCAN_MANIFEST_PARALLELISM.key());
validateIfPresent(table.options(),
CoreOptions.SCAN_PLAN_SORT_PARTITION.key());
if (table instanceof FallbackReadFileStoreTable) {
- validateEffectivePlanningTable(((FallbackReadFileStoreTable)
table).fallback());
+ validateEffectivePlanningTable(((FallbackReadFileStoreTable)
table).other());
}
if (table instanceof DelegatedFileStoreTable) {
validateEffectivePlanningTable(((DelegatedFileStoreTable)
table).wrapped());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
index 1a2d0511a4c..c3deea13c63 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java
@@ -242,7 +242,8 @@ public class PaimonUtil {
for (PaimonPartitionCandidate candidate : candidates) {
PartitionEntry entry = candidate.partitionEntry;
Partition partition = new Partition(candidate.typedSpec,
entry.recordCount(),
- entry.fileSizeInBytes(), entry.fileCount(),
entry.lastFileCreationTime(), false);
+ entry.fileSizeInBytes(), entry.fileCount(),
entry.lastFileCreationTime(),
+ entry.totalBuckets(), false);
nameToPartitionItem.put(candidate.displayName,
candidate.partitionItem);
nameToPartition.put(candidate.displayName, partition);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
index 7707aed9a62..ebd0dcbf4fb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java
@@ -89,6 +89,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.OptionalInt;
+import java.util.OptionalLong;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
@@ -525,15 +526,18 @@ public class PaimonScanNode extends FileQueryScanNode {
}
Optional<List<RawFile>> optRawFiles =
dataSplit.convertToRawFiles();
Optional<List<DeletionFile>> optDeletionFiles =
dataSplit.deletionFiles();
- if (applyCountPushdown && dataSplit.mergedRowCountAvailable()) {
- splitStat.setMergedRowCount(dataSplit.mergedRowCount());
+ OptionalLong mergedRowCount = applyCountPushdown
+ ? dataSplit.mergedRowCount() : OptionalLong.empty();
+ if (applyCountPushdown && mergedRowCount.isPresent()) {
+ long count = mergedRowCount.getAsLong();
+ splitStat.setMergedRowCount(count);
PaimonSplit split = new PaimonSplit(dataSplit);
- split.setRowCount(dataSplit.mergedRowCount());
+ split.setRowCount(count);
if (partitionInfoMap != null) {
split.setPaimonPartitionValues(partitionInfoMap);
}
pushDownCountSplits.add(split);
- pushDownCountSum += dataSplit.mergedRowCount();
+ pushDownCountSum += count;
} else if (!forceJniScanner && !forceJniForSystemTable &&
supportNativeReader(optRawFiles)) {
if (ignoreSplitType ==
SessionVariable.IgnoreSplitType.IGNORE_NATIVE) {
continue;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java
index 6d75834b504..f6856c78e2f 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonReaderOptionsTest.java
@@ -39,6 +39,7 @@ import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.mockito.Mockito;
+import java.lang.reflect.Field;
import java.util.Collections;
import java.util.Map;
@@ -125,12 +126,12 @@ public class PaimonReaderOptionsTest {
FileStoreTable safeEmpty = (FileStoreTable)
PaimonReaderOptions.runtimeSafeTable(empty, 512);
FallbackReadFileStoreTable explicitThenEmpty =
(FallbackReadFileStoreTable)
PaimonReaderOptions.runtimeSafeTable(
- new FallbackReadFileStoreTable(explicit, empty), 512);
+ new FallbackReadFileStoreTable(explicit, empty, true),
512);
FallbackReadFileStoreTable emptyThenExplicit =
(FallbackReadFileStoreTable)
PaimonReaderOptions.runtimeSafeTable(
- new FallbackReadFileStoreTable(empty, explicit), 512);
+ new FallbackReadFileStoreTable(empty, explicit, true),
512);
FileStoreTable privileged = PrivilegedFileStoreTable.wrap(
- new FallbackReadFileStoreTable(explicit, empty),
+ new FallbackReadFileStoreTable(explicit, empty, true),
Mockito.mock(PrivilegeChecker.class), Identifier.create("db",
"table"));
Table normalizedDelegate =
PaimonReaderOptions.runtimeSafeTable(privileged, 512);
@@ -138,22 +139,36 @@ public class PaimonReaderOptionsTest {
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
Assertions.assertEquals("1", explicitThenEmpty.wrapped().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
- Assertions.assertEquals("256", explicitThenEmpty.fallback().options()
+ Assertions.assertEquals("256", explicitThenEmpty.other().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
Assertions.assertEquals("256", emptyThenExplicit.wrapped().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
- Assertions.assertEquals("1", emptyThenExplicit.fallback().options()
+ Assertions.assertEquals("1", emptyThenExplicit.other().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
Assertions.assertInstanceOf(FallbackReadFileStoreTable.class,
normalizedDelegate);
Assertions.assertEquals("256", ((FallbackReadFileStoreTable)
normalizedDelegate)
-
.fallback().options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
+
.other().options().get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
+ }
+
+ @Test
+ void testRuntimeCapPreservesPrimaryBranchPriority() throws Exception {
+ FileStoreTable main = newFileStoreTable("main", ImmutableMap.of(
+ CoreOptions.SCAN_PRIMARY_BRANCH.key(), "primary",
+ CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), "128"));
+ FileStoreTable primary = newFileStoreTable("primary",
Collections.emptyMap());
+ Table safe = PaimonReaderOptions.runtimeSafeTable(
+ new FallbackReadFileStoreTable(main, primary, false), 64);
+
+ Field wrappedFirst =
FallbackReadFileStoreTable.class.getDeclaredField("wrappedFirst");
+ wrappedFirst.setAccessible(true);
+ Assertions.assertFalse(wrappedFirst.getBoolean(safe));
}
@Test
void testSystemSourceKeepsFallbackAsOutermostPlanningDecorator() {
FileStoreTable main = newFileStoreTable("main",
Collections.emptyMap());
FileStoreTable fallback = newFileStoreTable("fallback",
Collections.emptyMap());
- FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main,
fallback);
+ FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main,
fallback, true);
FileStoreTable privileged = PrivilegedFileStoreTable.wrap(
pair, Mockito.mock(PrivilegeChecker.class),
Identifier.create("db", "privileged_fallback"));
@@ -200,7 +215,7 @@ public class PaimonReaderOptionsTest {
FileStoreTable main = newFileStoreTable("main",
Collections.emptyMap());
FileStoreTable fallback = newFileStoreTable(
"fallback", ImmutableMap.of("scan.manifest.parallelism", "0"));
- Table fallbackReadTable = new FallbackReadFileStoreTable(main,
fallback);
+ Table fallbackReadTable = new FallbackReadFileStoreTable(main,
fallback, true);
Assertions.assertThrows(IllegalArgumentException.class,
() -> PaimonScanParams.applyOptions(fallbackReadTable,
Collections.emptyMap()));
@@ -211,7 +226,7 @@ public class PaimonReaderOptionsTest {
FileStoreTable main = newFileStoreTable("main",
Collections.emptyMap());
FileStoreTable fallback = newFileStoreTable(
"fallback", ImmutableMap.of("scan.manifest.parallelism", "0"));
- Table fallbackReadTable = new FallbackReadFileStoreTable(main,
fallback);
+ Table fallbackReadTable = new FallbackReadFileStoreTable(main,
fallback, true);
Assertions.assertDoesNotThrow(() -> PaimonScanParams.applyOptions(
fallbackReadTable,
ImmutableMap.of("scan.manifest.parallelism", "1")));
@@ -225,13 +240,13 @@ public class PaimonReaderOptionsTest {
FileStoreTable main = newFileStoreTable("main",
Collections.emptyMap());
FileStoreTable fallback = newFileStoreTable("fallback",
ImmutableMap.of(
CoreOptions.SCAN_MANIFEST_PARALLELISM.key(),
String.valueOf(localCapacity + 1)));
- Table fallbackReadTable = new FallbackReadFileStoreTable(main,
fallback);
+ Table fallbackReadTable = new FallbackReadFileStoreTable(main,
fallback, true);
Table safeTable =
PaimonReaderOptions.runtimeSafeTable(fallbackReadTable);
Assertions.assertDoesNotThrow(() ->
PaimonReaderOptions.validateEffectiveTable(safeTable));
Assertions.assertEquals(String.valueOf(localCapacity),
- ((FallbackReadFileStoreTable) safeTable).fallback().options()
+ ((FallbackReadFileStoreTable) safeTable).other().options()
.get(CoreOptions.SCAN_MANIFEST_PARALLELISM.key()));
}
@@ -246,7 +261,7 @@ public class PaimonReaderOptionsTest {
Math.min(Runtime.getRuntime().availableProcessors(),
PaimonReaderOptions.MAX_MANIFEST_PARALLELISM),
PaimonReaderOptions.backendManifestParallelismCap(
- new FallbackReadFileStoreTable(main,
fallback)).getAsInt());
+ new FallbackReadFileStoreTable(main, fallback,
true)).getAsInt());
}
@Test
@@ -254,7 +269,7 @@ public class PaimonReaderOptionsTest {
FileStoreTable main = newFileStoreTable("privileged_main",
Collections.emptyMap());
FileStoreTable fallback = newFileStoreTable(
"privileged_fallback",
ImmutableMap.of("scan.manifest.parallelism", "0"));
- FileStoreTable fallbackReadTable = new
FallbackReadFileStoreTable(main, fallback);
+ FileStoreTable fallbackReadTable = new
FallbackReadFileStoreTable(main, fallback, true);
FileStoreTable privilegedTable = PrivilegedFileStoreTable.wrap(
fallbackReadTable,
Mockito.mock(PrivilegeChecker.class),
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
index 1e7f7f03b23..09bcee53985 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java
@@ -84,7 +84,7 @@ public class PaimonUtilTest {
}
private static PartitionEntry partitionEntry(BinaryRow partition, long
sequence) {
- return new PartitionEntry(partition, sequence, sequence, sequence,
sequence);
+ return new PartitionEntry(partition, sequence, sequence, sequence,
sequence, 1);
}
@Test
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
index 9cb649b02ac..92f8da8d3b6 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java
@@ -95,6 +95,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalLong;
@RunWith(MockitoJUnitRunner.class)
public class PaimonScanNodeTest {
@@ -147,6 +148,22 @@ public class PaimonScanNodeTest {
Assert.assertEquals(Optional.of(15_000L), ((PaimonSplit)
countStarSplits.get(0)).getRowCount());
}
+ @Test
+ public void testNonCountScanDoesNotComputeMergedRowCount() throws
UserException {
+ PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new
TupleId(3), sv));
+
node.setSource(mockPaimonSourceWithPartitionKeys(Collections.<String>emptyList()));
+ DataSplit dataSplit = mockCountDataSplit("ordinary.parquet", 1_000);
+ Mockito.clearInvocations(dataSplit);
+
Mockito.doReturn(Collections.singletonList(dataSplit)).when(node).getPaimonSplitFromAPI();
+ Mockito.when(sv.isForceJniScanner()).thenReturn(true);
+ Mockito.when(sv.getIgnoreSplitType()).thenReturn("NONE");
+
+ List<org.apache.doris.spi.Split> splits = node.getSplits(1);
+
+ Assert.assertEquals(1, splits.size());
+ Mockito.verify(dataSplit, Mockito.never()).mergedRowCount();
+ }
+
@Test
public void testIncrementalBinlogCountStarDoesNotUsePhysicalRowCount()
throws UserException {
PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new
TupleId(3), sv));
@@ -1438,8 +1455,7 @@ public class PaimonScanNodeTest {
Collections.<String>emptyList());
DataSplit dataSplit = Mockito.mock(DataSplit.class);
Mockito.when(dataSplit.rowCount()).thenReturn(rowCount);
- Mockito.when(dataSplit.mergedRowCountAvailable()).thenReturn(true);
- Mockito.when(dataSplit.mergedRowCount()).thenReturn(rowCount);
+
Mockito.when(dataSplit.mergedRowCount()).thenReturn(OptionalLong.of(rowCount));
Mockito.when(dataSplit.partition()).thenReturn(BinaryRow.singleColumn(1));
Mockito.when(dataSplit.dataFiles()).thenReturn(Collections.singletonList(dataFileMeta));
Mockito.when(dataSplit.convertToRawFiles()).thenReturn(Optional.empty());
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java
b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java
index b7ec520af02..f6752074f0a 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/PaimonTableValuedFunctionTest.java
@@ -73,7 +73,7 @@ public class PaimonTableValuedFunctionTest {
Mockito.when(fallback.schema()).thenReturn(schema);
Mockito.when(main.options()).thenReturn(Collections.emptyMap());
Mockito.when(fallback.options()).thenReturn(Collections.emptyMap());
- FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main,
fallback);
+ FallbackReadFileStoreTable pair = new FallbackReadFileStoreTable(main,
fallback, true);
FileStoreTable privileged = PrivilegedFileStoreTable.wrap(
pair, Mockito.mock(PrivilegeChecker.class),
Identifier.create("db", "table"));
diff --git a/fe/pom.xml b/fe/pom.xml
index 270f98aaf96..ef84e3e2145 100644
--- a/fe/pom.xml
+++ b/fe/pom.xml
@@ -388,7 +388,7 @@ under the License.
<quartz.version>2.3.2</quartz.version>
<aircompressor.version>2.0.3</aircompressor.version>
<!-- paimon -->
- <paimon.version>1.3.1</paimon.version>
+ <paimon.version>1.4.2</paimon.version>
<disruptor.version>3.4.4</disruptor.version>
<!-- arrow flight sql -->
<arrow.vector.classifier>shade-format-flatbuffers</arrow.vector.classifier>
diff --git
a/regression-test/data/external_table_p0/paimon/paimon_system_table.out
b/regression-test/data/external_table_p0/paimon/paimon_system_table.out
index c2363693014..3d11cf0874d 100644
--- a/regression-test/data/external_table_p0/paimon/paimon_system_table.out
+++ b/regression-test/data/external_table_p0/paimon/paimon_system_table.out
@@ -1,6 +1,6 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !direct_query__snapshots_result --
-1 0 87d5f82b-5368-418d-91e6-58d01545e29e 9223372036854775807
APPEND 2024-07-11T16:01:57.425
manifest-list-102a9c07-edbc-4cde-a2bf-a4253a232e10-0
manifest-list-102a9c07-edbc-4cde-a2bf-a4253a232e10-1 \N 1 1
0 -9223372036854775808
+1 0 87d5f82b-5368-418d-91e6-58d01545e29e 9223372036854775807
APPEND 2024-07-11T16:01:57.425
manifest-list-102a9c07-edbc-4cde-a2bf-a4253a232e10-0
manifest-list-102a9c07-edbc-4cde-a2bf-a4253a232e10-1 \N 1 1
0 -9223372036854775808 \N
-- !paimon_snapshots_core_fields_direct_query --
1 0 87d5f82b-5368-418d-91e6-58d01545e29e 9223372036854775807
APPEND manifest-list-102a9c07-edbc-4cde-a2bf-a4253a232e10-0
manifest-list-102a9c07-edbc-4cde-a2bf-a4253a232e10-1 \N 1 1 0
@@ -34,6 +34,7 @@ total_record_count bigint Yes true \N NONE
delta_record_count bigint Yes true \N NONE
changelog_record_count bigint Yes true \N NONE
watermark bigint Yes true \N NONE
+next_row_id bigint Yes true \N NONE
-- !desc_direct_query_db_table --
snapshot_id bigint Yes true \N NONE
@@ -49,6 +50,7 @@ total_record_count bigint Yes true \N NONE
delta_record_count bigint Yes true \N NONE
changelog_record_count bigint Yes true \N NONE
watermark bigint Yes true \N NONE
+next_row_id bigint Yes true \N NONE
-- !desc_direct_query_table --
snapshot_id bigint Yes true \N NONE
@@ -64,12 +66,7 @@ total_record_count bigint Yes true \N NONE
delta_record_count bigint Yes true \N NONE
changelog_record_count bigint Yes true \N NONE
watermark bigint Yes true \N NONE
-
--- !files_at_first_snapshot --
-1
-
--- !files_at_latest_snapshot --
-3
+next_row_id bigint Yes true \N NONE
-- !files_without_options --
3
diff --git
a/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out
b/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out
index a3132a2c1b3..5cc72dd6fa1 100644
---
a/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out
+++
b/regression-test/data/external_table_p0/paimon/test_paimon_partition_table.out
@@ -1,29 +1,29 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !show_partition_sales_by_date --
-sale_date=19737 sale_date 2 2216 1
-sale_date=19738 sale_date 2 4217 2
-sale_date=19739 sale_date 1 2118 1
+sale_date=19737 sale_date 2 2100 1
+sale_date=19738 sale_date 2 3985 2
+sale_date=19739 sale_date 1 2002 1
-- !show_partition_sales_by_region --
-region=China-Beijing region 1 2425 1
-region=Japan-Tokyo region 1 2420 1
-region=USA-California region 1 2454 1
+region=China-Beijing region 1 2295 1
+region=Japan-Tokyo region 1 2290 1
+region=USA-California region 1 2324 1
-- !show_partition_sales_by_date_region --
-sale_date=19737/region=China-Beijing sale_date,region 1 2627
1
-sale_date=19737/region=Japan-Tokyo sale_date,region 1 2614
1
-sale_date=19737/region=USA-California sale_date,region 1 2655
1
-sale_date=19738/region=China-Shanghai sale_date,region 1 2636
1
-sale_date=19738/region=Japan-Osaka sale_date,region 1 2636
1
-sale_date=19738/region=USA-New York sale_date,region 1 2643
1
+sale_date=19737/region=China-Beijing sale_date,region 1 2485
1
+sale_date=19737/region=Japan-Tokyo sale_date,region 1 2472
1
+sale_date=19737/region=USA-California sale_date,region 1 2513
1
+sale_date=19738/region=China-Shanghai sale_date,region 1 2494
1
+sale_date=19738/region=Japan-Osaka sale_date,region 1 2494
1
+sale_date=19738/region=USA-New York sale_date,region 1 2501
1
-- !show_partition_events_by_hour --
-hour_partition=2024-01-15-10 hour_partition 2 2361 1
-hour_partition=2024-01-15-11 hour_partition 2 4515 2
-hour_partition=2024-01-15-14 hour_partition 2 2371 1
+hour_partition=2024-01-15-10 hour_partition 2 2245 1
+hour_partition=2024-01-15-11 hour_partition 2 4283 2
+hour_partition=2024-01-15-14 hour_partition 2 2255 1
-- !show_partition_logs_by_date_hierarchy --
-year_val=2024/month_val=1/day_val=15 year_val,month_val,day_val 2
2841 1
-year_val=2024/month_val=1/day_val=16 year_val,month_val,day_val 2
5323 2
-year_val=2024/month_val=1/day_val=17 year_val,month_val,day_val 1
2658 1
-year_val=2024/month_val=2/day_val=1 year_val,month_val,day_val 1
2686 1
+year_val=2024/month_val=1/day_val=15 year_val,month_val,day_val 2
2701 1
+year_val=2024/month_val=1/day_val=16 year_val,month_val,day_val 2
5042 2
+year_val=2024/month_val=1/day_val=17 year_val,month_val,day_val 1
2517 1
+year_val=2024/month_val=2/day_val=1 year_val,month_val,day_val 1
2546 1
diff --git
a/regression-test/data/paimon_write/test_paimon_write_append_only.out
b/regression-test/data/paimon_write/test_paimon_write_append_only.out
index f7e90eee87f..384cd097d91 100644
--- a/regression-test/data/paimon_write/test_paimon_write_append_only.out
+++ b/regression-test/data/paimon_write/test_paimon_write_append_only.out
@@ -21,10 +21,10 @@
6 default_partition \N
-- !ao_auto_partition_metadata --
-{2026-07-01} 3
-{2026-07-02} 1
-{2026-07-03} 1
-{null} 1
+dt=2026-07-01 3
+dt=2026-07-02 1
+dt=2026-07-03 1
+dt=__DEFAULT_PARTITION__ 1
-- !ao_empty --
1 \N
@@ -40,4 +40,4 @@
1 omitted-partition 2026-07-01
-- !ao_partition_default_metadata --
-{2026-07-01} 1
+dt=2026-07-01 1
diff --git
a/regression-test/data/paimon_write/test_paimon_write_schema_change.out
b/regression-test/data/paimon_write/test_paimon_write_schema_change.out
index a7d17ed3786..5931eef1ace 100644
--- a/regression-test/data/paimon_write/test_paimon_write_schema_change.out
+++ b/regression-test/data/paimon_write/test_paimon_write_schema_change.out
@@ -355,20 +355,20 @@ small_col smallint Yes true \N
small integer
9 ivan 3000000000 9.90 900 after-bigint 9000 9
900 2026-07-07
-- !sc_append_partitions --
-{2026-07-01} 2
-{2026-07-02} 2
-{2026-07-03} 1
-{2026-07-04} 1
-{2026-07-05} 2
-{2026-07-06} 1
-{2026-07-07} 1
-{2026-07-08} 1
-{2026-07-09} 1
-{2026-07-10} 3
-{2026-07-11} 2
-{2026-07-12} 1
-{2026-07-13} 1
-{2026-07-14} 1
+dt=2026-07-01 2
+dt=2026-07-02 2
+dt=2026-07-03 1
+dt=2026-07-04 1
+dt=2026-07-05 2
+dt=2026-07-06 1
+dt=2026-07-07 1
+dt=2026-07-08 1
+dt=2026-07-09 1
+dt=2026-07-10 3
+dt=2026-07-11 2
+dt=2026-07-12 1
+dt=2026-07-13 1
+dt=2026-07-14 1
-- !sc_types_initial --
1 100 30000 2000000000 1.5 123456.78
diff --git
a/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy
b/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy
index 6eb1ae4e312..d6862887eb4 100644
--- a/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy
+++ b/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy
@@ -198,7 +198,8 @@ suite("paimon_system_table",
"p0,external,doris,external_docker,external_docker_
@incr('startSnapshotId'='${firstSnapshotId}',
'endSnapshotId'='${latestSnapshotId}')
"""
- assertEquals(2L, ((Number) incrementalPartitions[0][0]).longValue())
+ // The partitions system table reports the aggregate at the selected
end snapshot.
+ assertEquals(3L, ((Number) incrementalPartitions[0][0]).longValue())
List<Long> incrementalRoIds = sql("""
select id from ${multiSnapshotTable}\$ro
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
