This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 720f12303f6 HDDS-13786. Flush BucketInfo changes corresponding in case
an empty directory is purged (#9149)
720f12303f6 is described below
commit 720f12303f60b191a6ac0bf5a2faa9ebeade7764
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Tue Oct 14 11:47:46 2025 -0400
HDDS-13786. Flush BucketInfo changes corresponding in case an empty
directory is purged (#9149)
---
.../key/OMDirectoriesPurgeRequestWithFSO.java | 1 +
.../ozone/om/request/key/OMKeyPurgeRequest.java | 23 +++++++---------------
.../key/OMDirectoriesPurgeResponseWithFSO.java | 6 ++++++
.../TestOMDirectoriesPurgeRequestAndResponse.java | 22 ++++++++++++++-------
4 files changed, 29 insertions(+), 23 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
index adbbf58a98f..ed6185141d6 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
@@ -205,6 +205,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
bucketNameInfo.getVolumeName(), bucketNameInfo.getBucketName());
if (omBucketInfo != null && omBucketInfo.getObjectID() ==
path.getBucketId()) {
omBucketInfo.purgeSnapshotUsedNamespace(1);
+ volBucketInfoMap.put(Pair.of(omBucketInfo.getVolumeName(),
omBucketInfo.getBucketName()), omBucketInfo);
}
numDirsDeleted++;
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
index b543c3cfbf6..5fe932c4a09 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
@@ -127,9 +127,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
if (keysToBePurgedList.isEmpty() && renamedKeysToBePurged.isEmpty()) {
OMException oe = new OMException("No keys found to be purged or renamed
in the request.",
OMException.ResultCodes.KEY_DELETION_ERROR);
- if (LOG.isDebugEnabled()) {
-
AUDIT.logWriteFailure(ozoneManager.buildAuditMessageForFailure(OMSystemAction.KEY_DELETION,
null, oe));
- }
+
AUDIT.logWriteFailure(ozoneManager.buildAuditMessageForFailure(OMSystemAction.KEY_DELETION,
null, oe));
return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, oe));
}
@@ -146,6 +144,9 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
// last purge transaction when running for AOS
deletingServiceMetrics.setLastAOSTransactionInfo(transactionInfo);
}
+ List<OmBucketInfo> bucketInfoList =
updateBucketSize(purgeKeysRequest.getBucketPurgeKeysSizeList(),
+ omMetadataManager);
+
if (LOG.isDebugEnabled()) {
Map<String, String> auditParams = new LinkedHashMap<>();
if (fromSnapshotInfo != null) {
@@ -161,21 +162,12 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
}
AUDIT.logWriteSuccess(ozoneManager.buildAuditMessageForSuccess(OMSystemAction.KEY_DELETION,
auditParams));
}
+ return new OMKeyPurgeResponse(omResponse.build(), keysToBePurgedList,
renamedKeysToBePurged, fromSnapshotInfo,
+ keysToUpdateList, bucketInfoList);
} catch (IOException e) {
- if (LOG.isDebugEnabled()) {
-
AUDIT.logWriteFailure(ozoneManager.buildAuditMessageForFailure(OMSystemAction.KEY_DELETION,
null, e));
- }
+
AUDIT.logWriteFailure(ozoneManager.buildAuditMessageForFailure(OMSystemAction.KEY_DELETION,
null, e));
return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, e));
}
- try {
- List<OmBucketInfo> bucketInfoList =
updateBucketSize(purgeKeysRequest.getBucketPurgeKeysSizeList(),
- omMetadataManager);
- return new OMKeyPurgeResponse(omResponse.build(),
- keysToBePurgedList, renamedKeysToBePurged, fromSnapshotInfo,
keysToUpdateList, bucketInfoList);
- } catch (OMException oe) {
-
AUDIT.logWriteFailure(ozoneManager.buildAuditMessageForFailure(OMSystemAction.KEY_DELETION,
null, oe));
- return new OMKeyPurgeResponse(createErrorOMResponse(omResponse, oe));
- }
}
private List<OmBucketInfo> updateBucketSize(List<BucketPurgeKeysSize>
bucketPurgeKeysSizeList,
@@ -226,5 +218,4 @@ private List<OmBucketInfo>
updateBucketSize(List<BucketPurgeKeysSize> bucketPurg
mergeOmLockDetails(omMetadataManager.getLock().releaseWriteLocks(BUCKET_LOCK,
bucketKeyList));
}
}
-
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
index 2ca04c4aae1..1cf078bca0a 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
@@ -23,6 +23,7 @@
import static org.apache.hadoop.ozone.om.codec.OMDBDefinition.FILE_TABLE;
import static
org.apache.hadoop.ozone.om.codec.OMDBDefinition.SNAPSHOT_INFO_TABLE;
+import com.google.common.annotations.VisibleForTesting;
import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.List;
@@ -186,4 +187,9 @@ public void processPaths(
}
}
}
+
+ @VisibleForTesting
+ public Map<Pair<String, String>, OmBucketInfo> getVolBucketInfoMap() {
+ return volBucketInfoMap;
+ }
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
index ff3fc5f31b9..54087fa64dc 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
@@ -185,8 +185,8 @@ private OMRequest createPurgeKeysRequest(String
fromSnapshot, String purgeDelete
volumeId, bucketId, purgeDeletedDir, subFiles, subDirs);
purgePathRequestList.add(request);
return createPurgeKeysRequest(fromSnapshot, purgePathRequestList,
Collections.singletonList(
-
BucketNameInfo.newBuilder().setVolumeName(volumeName).setBucketName(bucketName)
- .setBucketId(bucketId).setVolumeId(volumeId).buildPartial()));
+
BucketNameInfo.newBuilder().setVolumeName(bucketInfo.getVolumeName()).setBucketName(bucketInfo.getBucketName())
+ .setBucketId(bucketId).setVolumeId(volumeId).build()));
}
private PurgePathRequest wrapPurgeRequest(
@@ -356,8 +356,10 @@ public void testBucketLockWithPurgeDirectory() throws
Exception {
}
@ParameterizedTest
- @CsvSource(value = {"false,false", "false,true", "true,false", "true,true"})
- public void testDirectoryPurge(boolean fromSnapshot, boolean purgeDirectory)
throws Exception {
+ @CsvSource(value = {"false,false,0", "false,true,0", "true,false,0",
"true,true,0",
+ "false,false,10", "false,true,10", "true,false,10", "true,true,10"})
+ public void testDirectoryPurge(boolean fromSnapshot, boolean purgeDirectory,
int numberOfSubEntries)
+ throws Exception {
when(ozoneManager.getDefaultReplicationConfig())
.thenReturn(RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE));
String bucket = "bucket" + RandomUtils.secure().randomInt();
@@ -366,6 +368,7 @@ public void testDirectoryPurge(boolean fromSnapshot,
boolean purgeDirectory) thr
omMetadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED);
String bucketKey = omMetadataManager.getBucketKey(volumeName, bucket);
OmBucketInfo bucketInfo =
omMetadataManager.getBucketTable().get(bucketKey);
+ long purgeUsedNamespaceCountBeforePurge =
bucketInfo.getSnapshotUsedNamespace();
OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder()
.setName("dir1")
.setCreationTime(Time.now())
@@ -382,7 +385,7 @@ public void testDirectoryPurge(boolean fromSnapshot,
boolean purgeDirectory) thr
List<String> subDirKeys = new ArrayList<>();
List<String> deletedSubDirKeys = new ArrayList<>();
List<String> deletedSubFiles = new ArrayList<>();
- for (int id = 1; id < 10; id++) {
+ for (int id = 0; id < numberOfSubEntries; id++) {
OmDirectoryInfo subdir = new OmDirectoryInfo.Builder()
.setName("subdir" + id)
.setCreationTime(Time.now())
@@ -427,11 +430,16 @@ public void testDirectoryPurge(boolean fromSnapshot,
boolean purgeDirectory) thr
OMRequest omRequest = createPurgeKeysRequest(snapshotInfo == null ? null :
snapshotInfo.getTableKey(),
purgeDirectory ? deletedDirKey : null, subDirs, subFiles, bucketInfo);
OMRequest preExecutedRequest = preExecute(omRequest);
- OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest =
- new OMDirectoriesPurgeRequestWithFSO(preExecutedRequest);
+ OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest = new
OMDirectoriesPurgeRequestWithFSO(preExecutedRequest);
OMDirectoriesPurgeResponseWithFSO omClientResponse =
(OMDirectoriesPurgeResponseWithFSO) omKeyPurgeRequest
.validateAndUpdateCache(ozoneManager, 100L);
performBatchOperationCommit(omClientResponse);
+ OmBucketInfo updatedBucketInfo = purgeDirectory || numberOfSubEntries > 0 ?
+ omMetadataManager.getBucketTable().getSkipCache(bucketKey) :
omMetadataManager.getBucketTable().get(bucketKey);
+ long currentSnapshotUsedNamespace =
updatedBucketInfo.getSnapshotUsedNamespace();
+
+ assertEquals(purgeUsedNamespaceCountBeforePurge - (purgeDirectory ? 1 : 0)
+
+ (2 * (long)numberOfSubEntries), currentSnapshotUsedNamespace);
try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshot = fromSnapshot ?
ozoneManager.getOmSnapshotManager()
.getSnapshot(snapshotInfo.getSnapshotId()) : null) {
OMMetadataManager metadataManager = fromSnapshot ?
snapshot.get().getMetadataManager() :
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
