This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 099464e7ec [common] Version the memory-mode cache key in CachingFileIO
(#9642)
099464e7ec is described below
commit 099464e7ec17bae65af40ae2725b6d3b8e58b746
Author: YangJie <[email protected]>
AuthorDate: Fri Sep 11 03:10:24 2026 -0400
[common] Version the memory-mode cache key in CachingFileIO (#9642)
---
.../java/org/apache/paimon/utils/FileType.java | 14 +++++-
.../apache/paimon/fs/cache/CachingFileIOTest.java | 57 ++++++++++++++++++++++
.../java/org/apache/paimon/utils/FileTypeTest.java | 32 ++++++++++++
3 files changed, 101 insertions(+), 2 deletions(-)
diff --git a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
index 46c767bb24..9b30c864c1 100644
--- a/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
+++ b/paimon-common/src/main/java/org/apache/paimon/utils/FileType.java
@@ -114,8 +114,18 @@ public enum FileType {
/** Returns {@code true} if the file is mutable and should not be cached.
*/
public static boolean isMutable(Path filePath) {
- String name = filePath.getName();
- return "EARLIEST".equals(name) || "LATEST".equals(name);
+ String name = unwrapTempFileName(filePath.getName());
+ // Files rewritten in place under a stable path: caching them by path
keeps serving the
+ // pre-overwrite content (and a len+mtime key still collides when a
rewrite lands at the
+ // same size within the same clock second). Hint files, consumer and
service progress
+ // files, replaceable tags and _SUCCESS all go through
overwriteFileUtf8.
+ return "EARLIEST".equals(name)
+ || "LATEST".equals(name)
+ || "_SUCCESS".equals(name)
+ || name.endsWith("_SUCCESS")
+ || name.startsWith(CONSUMER_PREFIX)
+ || name.startsWith(SERVICE_PREFIX)
+ || name.startsWith(TAG_PREFIX);
}
/**
diff --git
a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java
b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java
index ded23667a9..05ddd7cad0 100644
---
a/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java
+++
b/paimon-common/src/test/java/org/apache/paimon/fs/cache/CachingFileIOTest.java
@@ -94,6 +94,63 @@ class CachingFileIOTest {
MockFileIO.resetGlobalInputStreamCalls();
}
+ @Test
+ void testMemoryModeDoesNotCacheInPlaceOverwrittenFiles() throws
IOException {
+ MockFileIO delegate = new MockFileIO();
+ CachingFileIO cachingIO =
+ newCachingFileIO(
+ delegate,
+ new LocalMemoryCacheManager(Long.MAX_VALUE, 64),
+ EnumSet.of(FileType.META),
+ 64);
+ Path consumer = new Path("consumer-1");
+
+ // consumer-* is written in place by overwriteFileUtf8, so it is
mutable and bypasses the
+ // cache: each read reaches the delegate and sees the current content.
+ delegate.addFile("consumer-1", "v1cc".getBytes());
+ try (SeekableInputStream in = cachingIO.newInputStream(consumer)) {
+ assertThat(in).isNotInstanceOf(CachingSeekableInputStream.class);
+ byte[] buf = new byte[4];
+ in.read(buf, 0, 4);
+ assertThat(new String(buf)).isEqualTo("v1cc");
+ }
+
assertThat(delegate.newInputStreamCallCount("consumer-1")).isEqualTo(1);
+
+ delegate.addFile("consumer-1", "v2cc".getBytes());
+ try (SeekableInputStream in = cachingIO.newInputStream(consumer)) {
+ byte[] buf = new byte[4];
+ in.read(buf, 0, 4);
+ assertThat(new String(buf)).isEqualTo("v2cc");
+ }
+ // never cached: the overwrite is visible and the delegate was opened
again
+
assertThat(delegate.newInputStreamCallCount("consumer-1")).isEqualTo(2);
+ }
+
+ @Test
+ void testMemoryModeImmutableCacheHitsDoNotRestatDelegate() throws
IOException {
+ MockFileIO delegate = new MockFileIO();
+ CachingFileIO cachingIO =
+ newCachingFileIO(
+ delegate,
+ new LocalMemoryCacheManager(Long.MAX_VALUE, 64),
+ EnumSet.of(FileType.META),
+ 64);
+ Path snapshot = new Path("snapshot-1");
+ delegate.addFile("snapshot-1", "0123456789abcdef".getBytes());
+
+ for (int i = 0; i < 3; i++) {
+ try (SeekableInputStream in = cachingIO.newInputStream(snapshot)) {
+ assertThat(readAll(in,
16)).isEqualTo("0123456789abcdef".getBytes());
+ }
+ }
+
+ // An immutable file keeps the path-only memory key: opened once, then
served from cache.
+ // Its size is resolved lazily and remembered, so repeated hits do not
re-stat the
+ // delegate the way moving getFileStatus onto every open would.
+
assertThat(delegate.newInputStreamCallCount("snapshot-1")).isEqualTo(1);
+ assertThat(delegate.getFileStatusCallCount("snapshot-1")).isEqualTo(1);
+ }
+
@Test
void testCreateBlobPresignedUrlDelegates() throws IOException {
FileIO delegate = mock(FileIO.class);
diff --git
a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
index d946f71bb7..d043d6bf97 100644
--- a/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
+++ b/paimon-common/src/test/java/org/apache/paimon/utils/FileTypeTest.java
@@ -31,6 +31,38 @@ public class FileTypeTest {
private static final String TABLE_ROOT =
"hdfs://cluster/warehouse/db.db/table";
+ // ===== mutable files (bypass the cache) =====
+
+ @Test
+ public void testIsMutable() {
+ // Files overwritten in place under a stable path must bypass the
cache.
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/snapshot/EARLIEST"))).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/snapshot/LATEST"))).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/consumer/consumer-myGroup")))
+ .isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/service/service-primary-key-lookup")))
+ .isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/tag/tag-myTag"))).isTrue();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/dt=2024-01-01/bucket-0/_SUCCESS")))
+ .isTrue();
+ // A temp rewrite of a mutable file is still mutable.
+ assertThat(
+ FileType.isMutable(
+ new Path(
+ TABLE_ROOT
+ +
"/consumer/.consumer-myGroup."
+ + UUID.randomUUID()
+ + ".tmp")))
+ .isTrue();
+
+ // Write-once files stay cacheable: a new version lands under a new
name.
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/snapshot/snapshot-1"))).isFalse();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/schema/schema-0"))).isFalse();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/manifest/manifest-a1b2c3d4-0")))
+ .isFalse();
+ assertThat(FileType.isMutable(new Path(TABLE_ROOT +
"/bucket-0/data-abc.orc"))).isFalse();
+ }
+
// ===== META files =====
@Test
