This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 27ec170ef7 [iceberg] Skip an already-deleted manifest list when
expiring old metadata (#9343)
27ec170ef7 is described below
commit 27ec170ef7aab1cb87d882af1aab112d29d3e139
Author: Lucas Gameiro <[email protected]>
AuthorDate: Sun Aug 23 17:21:28 2026 +0200
[iceberg] Skip an already-deleted manifest list when expiring old metadata
(#9343)
---
.../paimon/iceberg/IcebergCommitCallback.java | 7 ++++
.../paimon/iceberg/IcebergCompatibilityTest.java | 48 ++++++++++++++++++++++
2 files changed, 55 insertions(+)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
index f8a41f56f2..3f4532639f 100644
---
a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
+++
b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java
@@ -1608,6 +1608,13 @@ public class IcebergCommitCallback implements
CommitCallback, TagCallback {
}
expiredManifestLists.add(listName);
+ // A retained metadata JSON can reference a list an earlier
rebuild already
+ // deleted. Reading it must not fail: we only open it to
delete what it
+ // points at, and that earlier pass already did so.
+ if (!table.fileIO().exists(listPath)) {
+ continue;
+ }
+
for (IcebergManifestFileMeta meta :
manifestList.read(listName)) {
String metaName = new Path(meta.manifestPath()).getName();
if (expiredManifestFileMetas.contains(metaName)) {
diff --git
a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
index 6d1d9fc886..ca90e7a608 100644
---
a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java
@@ -463,6 +463,54 @@ public class IcebergCompatibilityTest {
commit.close();
}
+ @Test
+ public void testExpireAllBeforeSkipsAlreadyDeletedManifestList() throws
Exception {
+ RowType rowType =
+ RowType.of(
+ new DataType[] {DataTypes.INT(), DataTypes.INT()}, new
String[] {"k", "v"});
+ FileStoreTable table =
+ createPaimonTable(rowType, Collections.emptyList(),
Collections.emptyList(), -1);
+
+ String commitUser = UUID.randomUUID().toString();
+ TableWriteImpl<?> write = table.newWrite(commitUser);
+ TableCommitImpl commit = table.newCommit(commitUser);
+
+ write.write(GenericRow.of(1, 10));
+ commit.commit(1, write.prepareCommit(false, 1));
+ write.write(GenericRow.of(2, 20));
+ commit.commit(2, write.prepareCommit(false, 2));
+
+ IcebergPathFactory pathFactory =
+ new IcebergPathFactory(new Path(table.location(), "metadata"));
+ long latest = table.latestSnapshot().get().id();
+
+ // v(latest - 1) is retained by previous-versions-max = 1, but an
earlier from-scratch
+ // rebuild deletes the manifest lists of every version below the one
it rebuilds at.
+ // Reproduce that state: the metadata JSON survives, the list it
points at does not.
+ Path retainedMetadataPath = pathFactory.toMetadataPath(latest - 1);
+ IcebergMetadata retained = IcebergMetadata.fromPath(table.fileIO(),
retainedMetadataPath);
+ Path danglingListPath = new
Path(retained.currentSnapshot().manifestList());
+ table.fileIO().deleteQuietly(danglingListPath);
+ assertThat(table.fileIO().exists(retainedMetadataPath)).isTrue();
+ assertThat(table.fileIO().exists(danglingListPath)).isFalse();
+
+ // Dropping the base metadata sends the next commit down the
from-scratch path, which
+ // calls expireAllBefore and so walks the retained JSON above.
+ table.fileIO().deleteQuietly(pathFactory.toMetadataPath(latest));
+
+ write.write(GenericRow.of(3, 30));
+ commit.commit(3, write.prepareCommit(false, 3));
+
+ // The rebuild completed and published a usable Iceberg head.
+ Path rebuiltMetadataPath =
pathFactory.toMetadataPath(table.latestSnapshot().get().id());
+ assertThat(table.fileIO().exists(rebuiltMetadataPath)).isTrue();
+ assertThat(getIcebergResult())
+ .containsExactlyInAnyOrder("Record(1, 10)", "Record(2, 20)",
"Record(3, 30)");
+
+ write.close();
+ commit.close();
+ }
+
@Test
public void testCommitAfterRollbackDoesNotDuplicateSchemas() throws
Exception {
RowType rowType =
