This is an automated email from the ASF dual-hosted git repository.
ivandika pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1184aff2f26 HDDS-13595. ListKeys should not overwrite entry if
non-null cache entry exists (#8961)
1184aff2f26 is described below
commit 1184aff2f26f5627de2f1ce469ce27afbd0c989d
Author: Ivan Andika <[email protected]>
AuthorDate: Tue Sep 2 18:12:26 2025 +0800
HDDS-13595. ListKeys should not overwrite entry if non-null cache entry
exists (#8961)
---
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 8 ++-
.../hadoop/ozone/om/TestOmMetadataManager.java | 73 +++++++++++++++++++++-
.../ozone/om/request/OMRequestTestUtils.java | 12 ++--
3 files changed, 83 insertions(+), 10 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
index 48988192b2d..4832c0cd041 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java
@@ -1028,8 +1028,6 @@ public ListKeysResult listKeys(String volumeName, String
bucketName,
} else {
seekPrefix = getBucketKey(volumeName, bucketName) + OM_KEY_PREFIX;
}
- int currentCount = 0;
-
TreeMap<String, OmKeyInfo> cacheKeyMap = new TreeMap<>();
Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> iterator =
@@ -1057,6 +1055,8 @@ public ListKeysResult listKeys(String volumeName, String
bucketName,
cacheKeyMap.put(key, omKeyInfo);
}
}
+
+ int currentCount = 0;
long readFromRDbStartNs, readFromRDbStopNs = 0;
// Get maxKeys from DB if it has.
try (TableIterator<String, ? extends KeyValue<String, OmKeyInfo>>
@@ -1075,7 +1075,9 @@ public ListKeysResult listKeys(String volumeName, String
bucketName,
CacheValue<OmKeyInfo> cacheValue =
keyTable.getCacheValue(new CacheKey<>(kv.getKey()));
if (cacheValue == null || cacheValue.getCacheValue() != null) {
- cacheKeyMap.put(kv.getKey(), kv.getValue());
+ // We use putIfAbsent since cache entry should be more up-to-date
and should not be overwritten
+ // by the outdated DB entry
+ cacheKeyMap.putIfAbsent(kv.getKey(), kv.getValue());
currentCount++;
}
} else {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
index e038812cac6..bebc5880788 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestOmMetadataManager.java
@@ -603,6 +603,73 @@ public void testListKeysWithFewDeleteEntriesInCache()
throws Exception {
}
+ @Test
+ public void testListKeysWithEntriesInCacheAndDB() throws Exception {
+ String volumeNameA = "volumeA";
+ String ozoneBucket = "ozoneBucket";
+
+ // Create volumes and bucket.
+ OMRequestTestUtils.addVolumeToDB(volumeNameA, omMetadataManager);
+
+ addBucketsToCache(volumeNameA, ozoneBucket);
+
+ String prefixKeyA = "key-a";
+ TreeMap<String, OmKeyInfo> keyAMap = new TreeMap<>();
+
+ for (int i = 1; i <= 100; i++) {
+ if (i % 2 == 0) {
+ // Add to DB
+ addKeysToOM(volumeNameA, ozoneBucket, prefixKeyA + i, i);
+
+ String key = omMetadataManager.getOzoneKey(volumeNameA,
+ ozoneBucket, prefixKeyA + i);
+ // Key is overwritten in cache (with higher updateID),
+ // but the cache has not been flushed to the DB
+ OmKeyInfo overwriteKey =
OMRequestTestUtils.createOmKeyInfo(volumeNameA, ozoneBucket, prefixKeyA + i,
+ RatisReplicationConfig.getInstance(ONE)).setUpdateID(100L).build();
+ omMetadataManager.getKeyTable(getDefaultBucketLayout()).addCacheEntry(
+ new CacheKey<>(key),
+ CacheValue.get(100L, overwriteKey));
+ keyAMap.put(prefixKeyA + i, overwriteKey);
+ } else {
+ // Add to cache
+ OmKeyInfo omKeyInfo = addKeysToOM(volumeNameA, ozoneBucket, prefixKeyA
+ i, i);
+ keyAMap.put(prefixKeyA + i, omKeyInfo);
+ }
+ }
+
+ // Now list keys which match with prefixKeyA.
+ List<OmKeyInfo> omKeyInfoList =
+ omMetadataManager.listKeys(volumeNameA, ozoneBucket,
+ null, prefixKeyA, 1000).getKeys();
+
+ assertEquals(100, omKeyInfoList.size());
+
+ TreeMap<String, OmKeyInfo> currentKeys = new TreeMap<>();
+
+ for (OmKeyInfo omKeyInfo : omKeyInfoList) {
+ currentKeys.put(omKeyInfo.getKeyName(), omKeyInfo);
+ assertTrue(omKeyInfo.getKeyName().startsWith(prefixKeyA));
+ }
+
+ assertEquals(keyAMap, currentKeys);
+
+ omKeyInfoList =
+ omMetadataManager.listKeys(volumeNameA, ozoneBucket,
+ null, prefixKeyA, 100).getKeys();
+ assertEquals(100, omKeyInfoList.size());
+
+ omKeyInfoList =
+ omMetadataManager.listKeys(volumeNameA, ozoneBucket,
+ null, prefixKeyA, 98).getKeys();
+ assertEquals(98, omKeyInfoList.size());
+
+ omKeyInfoList =
+ omMetadataManager.listKeys(volumeNameA, ozoneBucket,
+ null, prefixKeyA, 1).getKeys();
+ assertEquals(1, omKeyInfoList.size());
+ }
+
/**
* Tests inner impl of listOpenFiles with different bucket types with and
* without pagination. NOTE: This UT does NOT test hsync here since the hsync
@@ -989,14 +1056,14 @@ private List<String> getMultipartKeyNames(
.collect(Collectors.toList());
}
- private void addKeysToOM(String volumeName, String bucketName,
+ private OmKeyInfo addKeysToOM(String volumeName, String bucketName,
String keyName, int i) throws Exception {
if (i % 2 == 0) {
- OMRequestTestUtils.addKeyToTable(false, volumeName, bucketName, keyName,
+ return OMRequestTestUtils.addKeyToTable(false, volumeName, bucketName,
keyName,
1000L, RatisReplicationConfig.getInstance(ONE), omMetadataManager);
} else {
- OMRequestTestUtils.addKeyToTableCache(volumeName, bucketName, keyName,
+ return OMRequestTestUtils.addKeyToTableCache(volumeName, bucketName,
keyName,
RatisReplicationConfig.getInstance(ONE),
omMetadataManager);
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
index 8cacc58da1c..40a2a5a73fb 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
@@ -204,11 +204,11 @@ public static void addKeyToTable(boolean openKeyTable,
String volumeName,
* @throws Exception
*/
@SuppressWarnings("parameterNumber")
- public static void addKeyToTable(boolean openKeyTable, String volumeName,
+ public static OmKeyInfo addKeyToTable(boolean openKeyTable, String
volumeName,
String bucketName, String keyName, long clientID,
ReplicationConfig replicationConfig,
OMMetadataManager omMetadataManager) throws Exception {
- addKeyToTable(openKeyTable, false, volumeName, bucketName, keyName,
+ return addKeyToTable(openKeyTable, false, volumeName, bucketName, keyName,
clientID, replicationConfig, 0L, omMetadataManager);
}
@@ -265,7 +265,7 @@ replicationConfig, new OmKeyLocationInfoGroup(version, new
ArrayList<>(), false)
* @throws Exception
*/
@SuppressWarnings("parameternumber")
- public static void addKeyToTable(boolean openKeyTable, boolean addToCache,
+ public static OmKeyInfo addKeyToTable(boolean openKeyTable, boolean
addToCache,
String volumeName, String bucketName, String keyName, long clientID,
ReplicationConfig replicationConfig, long trxnLogIndex,
OMMetadataManager omMetadataManager) throws Exception {
@@ -275,6 +275,8 @@ public static void addKeyToTable(boolean openKeyTable,
boolean addToCache,
addKeyToTable(openKeyTable, addToCache, omKeyInfo, clientID, trxnLogIndex,
omMetadataManager);
+
+ return omKeyInfo;
}
/**
@@ -480,7 +482,7 @@ public static void addPart(PartKeyInfo partKeyInfo,
* @param omMetadataManager
*/
@SuppressWarnings("parameterNumber")
- public static void addKeyToTableCache(String volumeName,
+ public static OmKeyInfo addKeyToTableCache(String volumeName,
String bucketName,
String keyName,
ReplicationConfig replicationConfig,
@@ -492,6 +494,8 @@ public static void addKeyToTableCache(String volumeName,
omMetadataManager.getKeyTable(getDefaultBucketLayout()).addCacheEntry(
new CacheKey<>(omMetadataManager.getOzoneKey(volumeName, bucketName,
keyName)), CacheValue.get(1L, omKeyInfo));
+
+ return omKeyInfo;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
