This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 527d30a2f09 HDDS-13760. Track last purge transactionInfo in memory to
track inflight purge request in KDS and DDS (#9116)
527d30a2f09 is described below
commit 527d30a2f0959994724ffc9d804292051f42e03c
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Wed Oct 8 05:36:53 2025 -0400
HDDS-13760. Track last purge transactionInfo in memory to track inflight
purge request in KDS and DDS (#9116)
---
.../hadoop/ozone/om/DeletingServiceMetrics.java | 24 ++++++++++++++++++++++
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 15 +++-----------
.../key/OMDirectoriesPurgeRequestWithFSO.java | 7 ++++++-
.../ozone/om/request/key/OMKeyPurgeRequest.java | 7 ++++++-
.../om/service/AbstractKeyDeletingService.java | 15 ++++++++++++++
.../ozone/om/service/DirectoryDeletingService.java | 2 ++
.../ozone/om/service/KeyDeletingService.java | 2 ++
7 files changed, 58 insertions(+), 14 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
index 09f097bd5a6..ec4a110a4f9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
@@ -20,6 +20,7 @@
import com.google.common.annotations.VisibleForTesting;
import java.time.Instant;
import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.metrics2.annotation.Metric;
import org.apache.hadoop.metrics2.annotation.Metrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
@@ -109,6 +110,17 @@ public final class DeletingServiceMetrics {
@Metric("Snapshot: No. of not reclaimable keys the last run")
private MutableGaugeLong snapKeysNotReclaimableLast;
+ /**
+ * Metric to track the term ID of the last key that was purged from the
+ * Active Object Store (AOS). This term ID represents the state of the
+ * most recent successful purge operation in the AOS. This value would be
used ensure that a background
+ * KeyDeletingService/DirectoryDeletingService doesn't start the next run
until the previous run has been flushed.
+ */
+ @Metric("Last Purge Key termIndex on Active Object Store")
+ private MutableGaugeLong lastAOSPurgeTermId;
+ @Metric("Last Purge Key transactionId on Active Object Store")
+ private MutableGaugeLong lastAOSPurgeTransactionId;
+
private DeletingServiceMetrics() {
this.registry = new MetricsRegistry(METRICS_SOURCE_NAME);
}
@@ -287,6 +299,18 @@ public long getSnapKeysNotReclaimableLast() {
return snapKeysNotReclaimableLast.value();
}
+ public synchronized TransactionInfo getLastAOSTransactionInfo() {
+ return TransactionInfo.valueOf(lastAOSPurgeTermId.value(),
lastAOSPurgeTransactionId.value());
+ }
+
+ public synchronized void setLastAOSTransactionInfo(TransactionInfo
transactionInfo) {
+ TransactionInfo previousTransactionInfo = getLastAOSTransactionInfo();
+ if (transactionInfo.compareTo(previousTransactionInfo) > 0) {
+ this.lastAOSPurgeTermId.set(transactionInfo.getTerm());
+
this.lastAOSPurgeTransactionId.set(transactionInfo.getTransactionIndex());
+ }
+ }
+
@VisibleForTesting
public void resetDirectoryMetrics() {
numDirsPurged.set(0);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index d48e15cf66d..8c50232355c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -2209,20 +2209,12 @@ private <T extends WithParentObjectId> DeleteKeysResult
gatherSubPathsWithIterat
String seekFileInDB = metadataManager.getOzonePathKey(volumeId, bucketId,
parentInfo.getObjectID(), "");
long consumedSize = 0;
- boolean processedSubPaths = false;
try (TableIterator<String, ? extends KeyValue<String, T>> iterator =
table.iterator(seekFileInDB)) {
while (iterator.hasNext() && remainingBufLimit > 0) {
KeyValue<String, T> entry = iterator.next();
- T withParentObjectId = entry.getValue();
final long objectSerializedSize = entry.getValueByteSize();
- if
(!OMFileRequest.isImmediateChild(withParentObjectId.getParentObjectID(),
- parentInfo.getObjectID())) {
- processedSubPaths = true;
- break;
- }
- if (!table.isExist(entry.getKey())) {
- continue;
- }
+ // No need to check the table again as the value in cache and iterator
would be same when directory
+ // deleting service runs.
if (remainingBufLimit - objectSerializedSize < 0) {
break;
}
@@ -2233,8 +2225,7 @@ private <T extends WithParentObjectId> DeleteKeysResult
gatherSubPathsWithIterat
consumedSize += objectSerializedSize;
}
}
- processedSubPaths = processedSubPaths || (!iterator.hasNext());
- return new DeleteKeysResult(keyInfos, consumedSize, processedSubPaths);
+ return new DeleteKeysResult(keyInfos, consumedSize, !iterator.hasNext());
}
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
index 4724410803e..a0d64234343 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
@@ -204,11 +204,16 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
deletingServiceMetrics.incrNumDirPurged(numDirsDeleted);
Map<String, String> auditParams = new LinkedHashMap<>();
+ TransactionInfo transactionInfo =
TransactionInfo.valueOf(context.getTermIndex());
if (fromSnapshotInfo != null) {
-
fromSnapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(context.getTermIndex()).toByteString());
+
fromSnapshotInfo.setLastTransactionInfo(transactionInfo.toByteString());
omMetadataManager.getSnapshotInfoTable().addCacheEntry(new
CacheKey<>(fromSnapshotInfo.getTableKey()),
CacheValue.get(context.getIndex(), fromSnapshotInfo));
auditParams.put(AUDIT_PARAM_SNAPSHOT_ID,
fromSnapshotInfo.getSnapshotId().toString());
+ } else {
+ // Update the deletingServiceMetrics with the transaction index to
indicate the
+ // last purge transaction when running for AOS
+ deletingServiceMetrics.setLastAOSTransactionInfo(transactionInfo);
}
auditParams.put(AUDIT_PARAM_DIRS_DELETED,
String.valueOf(numDirsDeleted));
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
index 66973b294c1..b649c1ef6ca 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
@@ -127,11 +127,16 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
// services.
try {
Map<String, String> auditParams = new LinkedHashMap<>();
+ TransactionInfo transactionInfo =
TransactionInfo.valueOf(context.getTermIndex());
if (fromSnapshotInfo != null) {
-
fromSnapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(context.getTermIndex()).toByteString());
+
fromSnapshotInfo.setLastTransactionInfo(transactionInfo.toByteString());
omMetadataManager.getSnapshotInfoTable().addCacheEntry(new
CacheKey<>(fromSnapshotInfo.getTableKey()),
CacheValue.get(context.getIndex(), fromSnapshotInfo));
auditParams.put(AUDIT_PARAM_SNAPSHOT_ID,
fromSnapshotInfo.getSnapshotId().toString());
+ } else {
+ // Update the deletingServiceMetrics with the transaction index to
indicate the
+ // last purge transaction when running for AOS
+ deletingServiceMetrics.setLastAOSTransactionInfo(transactionInfo);
}
auditParams.put(AUDIT_PARAM_KEYS_DELETED,
String.valueOf(numKeysDeleted));
auditParams.put(AUDIT_PARAM_RENAMED_KEYS_PURGED,
String.valueOf(renamedKeysToBePurged.size()));
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
index 3ee50ffd04f..5650ab7734e 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
@@ -19,12 +19,14 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ServiceException;
+import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.hadoop.hdds.utils.BackgroundService;
+import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
@@ -77,6 +79,19 @@ final boolean shouldRun() {
return !suspended.get() && getOzoneManager().isLeaderReady();
}
+ boolean isPreviousPurgeTransactionFlushed() throws IOException {
+ TransactionInfo lastAOSTransactionId = metrics.getLastAOSTransactionInfo();
+ TransactionInfo flushedTransactionId = TransactionInfo.readTransactionInfo(
+ getOzoneManager().getMetadataManager());
+ if (flushedTransactionId != null &&
lastAOSTransactionId.compareTo(flushedTransactionId) > 0) {
+ LOG.info("Skipping AOS processing since changes to deleted space of AOS
have not been flushed to disk " +
+ "last Purge Transaction: {}, Flushed Disk Transaction: {}",
lastAOSTransactionId,
+ flushedTransactionId);
+ return false;
+ }
+ return true;
+ }
+
/**
* Suspend the service.
*/
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
index 7abf1406d96..001e686455f 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/DirectoryDeletingService.java
@@ -714,6 +714,8 @@ public BackgroundTaskResult call() {
snapInfo);
return BackgroundTaskResult.EmptyTaskResult.newResult();
}
+ } else if (!isPreviousPurgeTransactionFlushed()) {
+ return BackgroundTaskResult.EmptyTaskResult.newResult();
}
try (UncheckedAutoCloseableSupplier<OmSnapshot> omSnapshot =
snapInfo == null ? null :
omSnapshotManager.getActiveSnapshot(snapInfo.getVolumeName(),
snapInfo.getBucketName(),
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
index a81f4674cfd..d7dba697b71 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
@@ -566,6 +566,8 @@ public BackgroundTaskResult call() {
" iteration.", snapInfo);
return EmptyTaskResult.newResult();
}
+ } else if (!isPreviousPurgeTransactionFlushed()) {
+ return EmptyTaskResult.newResult();
}
try (UncheckedAutoCloseableSupplier<OmSnapshot> omSnapshot =
snapInfo == null ? null :
omSnapshotManager.getActiveSnapshot(snapInfo.getVolumeName(),
snapInfo.getBucketName(),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
