This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c01a675323 [core] Limit index-only compact handling to data evolution
(#9001)
c01a675323 is described below
commit c01a675323e53e9d47e657fbfe9c4c73fd309106
Author: LsomeYeah <[email protected]>
AuthorDate: Mon Aug 3 21:14:12 2026 +0800
[core] Limit index-only compact handling to data evolution (#9001)
---
.../paimon/operation/FileStoreCommitImpl.java | 1 +
.../paimon/operation/commit/StrictModeChecker.java | 9 ++++++--
.../apache/paimon/table/sink/TableCommitTest.java | 25 ++++++++++++++++------
3 files changed, 26 insertions(+), 9 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
index 80f4ffba46..bc22370c47 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
@@ -233,6 +233,7 @@ public class FileStoreCommitImpl implements FileStoreCommit
{
commitUser,
scanSupplier,
indexManifestFile,
+ options.dataEvolutionEnabled(),
id))
.orElse(null);
this.conflictDetection = conflictDetectFactory.create(scanner);
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
b/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
index 52944ea713..00d94845ea 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/commit/StrictModeChecker.java
@@ -42,6 +42,7 @@ public class StrictModeChecker {
private final String commitUser;
private final Supplier<FileStoreScan> scanSupplier;
private final IndexManifestFile indexManifestFile;
+ private final boolean dataEvolutionEnabled;
private long strictModeLastSafeSnapshot;
@@ -50,11 +51,13 @@ public class StrictModeChecker {
String commitUser,
Supplier<FileStoreScan> scanSupplier,
IndexManifestFile indexManifestFile,
+ boolean dataEvolutionEnabled,
long strictModeLastSafeSnapshot) {
this.snapshotManager = snapshotManager;
this.commitUser = commitUser;
this.scanSupplier = scanSupplier;
this.indexManifestFile = indexManifestFile;
+ this.dataEvolutionEnabled = dataEvolutionEnabled;
this.strictModeLastSafeSnapshot = strictModeLastSafeSnapshot;
}
@@ -70,8 +73,10 @@ public class StrictModeChecker {
|| snapshot.commitKind() == CommitKind.OVERWRITE) {
boolean hasOverlap = hasOverlappedDataPartition(snapshot,
newPartitions);
// OVERWRITE may contain logical changes represented only by
deletion vectors,
- // while index-only COMPACT does not change table data.
- if (!hasOverlap && snapshot.commitKind() ==
CommitKind.OVERWRITE) {
+ // while an index-only COMPACT on a data evolution table does
not change data.
+ if (!hasOverlap
+ && (snapshot.commitKind() == CommitKind.OVERWRITE
+ || !dataEvolutionEnabled)) {
hasOverlap = hasOverlappedIndexPartition(snapshot,
newPartitions);
}
if (hasOverlap) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
index ee11348b31..d5a4237cc5 100644
---
a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java
@@ -542,8 +542,9 @@ public class TableCommitTest {
commit2.close();
}
- @Test
- public void testStrictModeIgnoresIndexOnlyCompact() throws Exception {
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ public void testStrictModeForIndexOnlyCompact(boolean
dataEvolutionEnabled) throws Exception {
String path = tempDir.toString();
RowType rowType =
RowType.of(
@@ -552,9 +553,13 @@ public class TableCommitTest {
Options options = new Options();
options.set(CoreOptions.PATH, path);
- options.set(CoreOptions.BUCKET, 1);
- options.set(CoreOptions.BUCKET_KEY, "k");
+ options.set(CoreOptions.BUCKET, dataEvolutionEnabled ? -1 : 1);
+ if (!dataEvolutionEnabled) {
+ options.set(CoreOptions.BUCKET_KEY, "k");
+ }
options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10);
+ options.set(CoreOptions.ROW_TRACKING_ENABLED, dataEvolutionEnabled);
+ options.set(CoreOptions.DATA_EVOLUTION_ENABLED, dataEvolutionEnabled);
TableSchema tableSchema =
SchemaUtils.forceCommit(
new SchemaManager(LocalFileIO.create(), new
Path(path)),
@@ -584,7 +589,6 @@ public class TableCommitTest {
TableWriteImpl<?> write2 = tableWithStrict.newWrite(user2);
TableCommitImpl commit2 = tableWithStrict.newCommit(user2);
write2.write(GenericRow.of(1, 1, 1L));
- write2.compact(pt1, 0, true);
IndexFileMeta btreeIndex =
new IndexFileMeta(
@@ -609,8 +613,15 @@ public class TableCommitTest {
Collections.singletonList(btreeIndex),
Collections.emptyList()))));
- assertThatCode(() -> commit2.commit(1, write2.prepareCommit(true, 1)))
- .doesNotThrowAnyException();
+ if (dataEvolutionEnabled) {
+ assertThatCode(() -> commit2.commit(1, write2.prepareCommit(true,
1)))
+ .doesNotThrowAnyException();
+ } else {
+ assertThatThrownBy(() -> commit2.commit(1,
write2.prepareCommit(true, 1)))
+ .isInstanceOf(RuntimeException.class)
+ .hasMessageContaining(
+ "Giving up committing as
commit.strict-mode.last-safe-snapshot is set.");
+ }
write1.close();
commit1.close();
