This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new c10da57361 [core] Clean up data file extra files (#9586)
c10da57361 is described below
commit c10da57361aa865c0ac6905e30fad1677e637000
Author: Zhang Jiawei <[email protected]>
AuthorDate: Fri Sep 4 15:22:51 2026 +0800
[core] Clean up data file extra files (#9586)
---
.../org/apache/paimon/append/AppendOnlyWriter.java | 4 +-
.../paimon/operation/FileStoreCommitImpl.java | 2 +-
.../apache/paimon/append/AppendOnlyWriterTest.java | 6 ++-
.../paimon/operation/FileStoreCommitTest.java | 61 ++++++++++++++++++++++
4 files changed, 69 insertions(+), 4 deletions(-)
diff --git
a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java
b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java
index 750bf0b918..906573ad73 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/AppendOnlyWriter.java
@@ -288,7 +288,7 @@ public class AppendOnlyWriter implements BatchRecordWriter,
MemoryOwner {
for (DataFileMeta file : compactAfter) {
// appendOnlyCompactManager will rewrite the file and no file
upgrade will occur, so we
// can directly delete the file in compactAfter.
- fileIO.deleteQuietly(pathFactory.toPath(file));
+ file.collectFiles(pathFactory).forEach(fileIO::deleteQuietly);
}
sinkWriter.close();
@@ -315,7 +315,7 @@ public class AppendOnlyWriter implements BatchRecordWriter,
MemoryOwner {
} finally {
// remove small files
for (DataFileMeta file : files) {
- fileIO.deleteQuietly(pathFactory.toPath(file));
+
file.collectFiles(pathFactory).forEach(fileIO::deleteQuietly);
}
}
}
diff --git
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
index 49a231821c..2d9c94ec72 100644
---
a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
+++
b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
@@ -714,7 +714,7 @@ public class FileStoreCommitImpl implements FileStoreCommit
{
dataFilesToDelete.addAll(commitMessage.compactIncrement().changelogFiles());
for (DataFileMeta file : dataFilesToDelete) {
- fileIO.deleteQuietly(dataPathFactory.toPath(file));
+
file.collectFiles(dataPathFactory).forEach(fileIO::deleteQuietly);
}
List<IndexFileMeta> indexFilesToDelete = new ArrayList<>();
diff --git
a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
index 5f8bd44c0b..3b2464d311 100644
---
a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyWriterTest.java
@@ -376,6 +376,8 @@ public class AppendOnlyWriterTest {
writer.write(row(j, String.format("%03d", j), PART));
}
writer.sync();
+ assertThat(Files.walk(tempDir).filter(Files::isRegularFile))
+ .anyMatch(path ->
path.getFileName().toString().endsWith(".index"));
// writer closed unexpectedly
writer.close();
@@ -1321,6 +1323,8 @@ public class AppendOnlyWriterTest {
long maxSeq = toCompact.get(size - 1).maxSequenceNumber();
Path path = pathFactory.newPath("compact-");
LocalFileIO.create().newOutputStream(path, false).close();
+ Path extraPath = new Path(path.getParent(), path.getName() + ".index");
+ LocalFileIO.create().newOutputStream(extraPath, false).close();
return DataFileMeta.forAppend(
path.getName(),
toCompact.stream().mapToLong(DataFileMeta::fileSize).sum(),
@@ -1350,7 +1354,7 @@ public class AppendOnlyWriterTest {
minSeq,
maxSeq,
toCompact.get(0).schemaId(),
- Collections.emptyList(),
+ Collections.singletonList(extraPath.getName()),
null,
FileSource.APPEND,
null,
diff --git
a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
index 2ba3cad249..008cf3131f 100644
---
a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
+++
b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
@@ -37,6 +37,7 @@ import org.apache.paimon.index.IndexFileMeta;
import org.apache.paimon.index.IndexPathFactory;
import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataFilePathFactory;
import org.apache.paimon.io.DataIncrement;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
@@ -1127,6 +1128,66 @@ public class FileStoreCommitTest {
assertThat(store.fileIO().exists(compactDeletedPath)).isTrue();
}
+ @Test
+ public void testAbortDataFileWithExtraFiles() throws Exception {
+ TestAppendFileStore store =
TestAppendFileStore.createAppendStore(tempDir, new HashMap<>());
+ BinaryRow partition = gen.getPartition(gen.next());
+ DataFilePathFactory pathFactory =
+ store.pathFactory().createDataFilePathFactory(partition, 0);
+
+ Path dataNewPath = pathFactory.newPath();
+ DataFileMeta dataNew = createDataFileWithExtraFile(store, dataNewPath,
false);
+ Path compactNewPath = new
Path(tempDir.resolve("external-compact-new.orc").toUri());
+ DataFileMeta compactNew = createDataFileWithExtraFile(store,
compactNewPath, true);
+
+ CommitMessage commitMessage =
+ new CommitMessageImpl(
+ partition,
+ 0,
+ store.options().bucket(),
+ new DataIncrement(
+ Collections.singletonList(dataNew),
+ Collections.emptyList(),
+ Collections.emptyList()),
+ new CompactIncrement(
+ Collections.emptyList(),
+ Collections.singletonList(compactNew),
+ Collections.emptyList()));
+
+ try (FileStoreCommitImpl commit = store.newCommit()) {
+ commit.abort(Collections.singletonList(commitMessage));
+ }
+
+ for (Path path : dataNew.collectFiles(pathFactory)) {
+ assertThat(store.fileIO().exists(path)).isFalse();
+ }
+ for (Path path : compactNew.collectFiles(pathFactory)) {
+ assertThat(store.fileIO().exists(path)).isFalse();
+ }
+ }
+
+ private static DataFileMeta createDataFileWithExtraFile(
+ TestAppendFileStore store, Path path, boolean external) throws
Exception {
+ store.fileIO().newOutputStream(path, false).close();
+ Path extraPath = new Path(path.getParent(), path.getName() + ".index");
+ store.fileIO().newOutputStream(extraPath, false).close();
+ return DataFileMeta.forAppend(
+ path.getName(),
+ 0,
+ 0,
+ EMPTY_STATS,
+ 0,
+ 0,
+ 0,
+ Collections.singletonList(extraPath.getName()),
+ null,
+ null,
+ null,
+ external ? path.toString() : null,
+ null,
+ null);
+ }
+
private static IndexFileMeta createIndexFile(
TestAppendFileStore store, Path path, boolean external) throws
Exception {
store.fileIO().newOutputStream(path, false).close();
