This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a1324b66a7 HDDS-11511. Introduce metrics in deletion services of OM
(#7377)
a1324b66a7 is described below
commit a1324b66a74e5fffdda7353d1d4ea752e0f69677
Author: Tejaskriya <[email protected]>
AuthorDate: Wed Jan 8 17:23:00 2025 +0530
HDDS-11511. Introduce metrics in deletion services of OM (#7377)
---
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 75 ++++++++++
.../hadoop/ozone/om/DeletingServiceMetrics.java | 163 +++++++++++++++++++++
.../org/apache/hadoop/ozone/om/OzoneManager.java | 6 +
.../key/OMDirectoriesPurgeRequestWithFSO.java | 13 ++
.../ozone/om/request/key/OMKeyPurgeRequest.java | 8 +-
.../om/service/AbstractKeyDeletingService.java | 7 +-
.../ozone/om/service/KeyDeletingService.java | 5 +
.../ozone/om/request/key/TestOMKeyRequest.java | 4 +
8 files changed, 279 insertions(+), 2 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
index 049db78cf6..6cced07848 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmSnapshotManager;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -109,6 +110,7 @@ public class TestDirectoryDeletingServiceWithFSO {
private static String volumeName;
private static String bucketName;
private static OzoneClient client;
+ private static DeletingServiceMetrics metrics;
@BeforeAll
public static void init() throws Exception {
@@ -140,6 +142,7 @@ public class TestDirectoryDeletingServiceWithFSO {
conf.setInt(OZONE_FS_ITERATE_BATCH_SIZE, 5);
fs = FileSystem.get(conf);
+ metrics = cluster.getOzoneManager().getDeletionMetrics();
}
@AfterAll
@@ -185,6 +188,7 @@ public class TestDirectoryDeletingServiceWithFSO {
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
+ metrics.resetDirectoryMetrics();
// Delete the appRoot, empty dir
fs.delete(appRoot, true);
@@ -197,6 +201,8 @@ public class TestDirectoryDeletingServiceWithFSO {
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 1);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
+ assertEquals(1, metrics.getNumDirsPurged());
+ assertEquals(1, metrics.getNumDirsSentForPurge());
try (TableIterator<?, ? extends Table.KeyValue<?, OmDirectoryInfo>>
iterator = dirTable.iterator()) {
@@ -255,6 +261,7 @@ public class TestDirectoryDeletingServiceWithFSO {
long preRunCount = dirDeletingService.getRunCount().get();
+ metrics.resetDirectoryMetrics();
// Delete the appRoot
fs.delete(appRoot, true);
@@ -270,6 +277,14 @@ public class TestDirectoryDeletingServiceWithFSO {
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 18);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 19);
+ assertEquals(15, metrics.getNumSubFilesSentForPurge());
+ assertEquals(15, metrics.getNumSubFilesMovedToDeletedTable());
+ assertEquals(19, metrics.getNumDirsPurged());
+ assertEquals(19, metrics.getNumDirsSentForPurge());
+ assertEquals(18, metrics.getNumSubDirsMovedToDeletedDirTable());
+ assertEquals(18, metrics.getNumSubDirsSentForPurge());
+
+
long elapsedRunCount = dirDeletingService.getRunCount().get() -
preRunCount;
assertThat(dirDeletingService.getRunCount().get()).isGreaterThan(1);
// Ensure dir deleting speed, here provide a backup value for safe CI
@@ -308,6 +323,7 @@ public class TestDirectoryDeletingServiceWithFSO {
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);
+ metrics.resetDirectoryMetrics();
// Delete the rootDir, which should delete all keys.
fs.delete(root, true);
@@ -321,7 +337,66 @@ public class TestDirectoryDeletingServiceWithFSO {
assertSubPathsCount(dirDeletingService::getMovedFilesCount, 3);
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 2);
assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 5);
+ assertEquals(5, metrics.getNumDirsSentForPurge());
+ assertEquals(5, metrics.getNumDirsPurged());
+ assertEquals(4, metrics.getNumSubDirsMovedToDeletedDirTable());
+ assertEquals(4, metrics.getNumSubDirsSentForPurge());
+ assertEquals(3, metrics.getNumSubFilesSentForPurge());
+ assertEquals(3, metrics.getNumSubFilesMovedToDeletedTable());
+
+ assertThat(dirDeletingService.getRunCount().get()).isGreaterThan(1);
+ }
+
+ /**
+ * Test to check the following scenario:
+ * A subdir gets marked for move in DirectoryDeletingService and
+ * marked for delete in
AbstractKeyDeletingService#optimizeDirDeletesAndSubmitRequest.
+ */
+ @Test
+ public void testDeleteWithLessDirsButMultipleLevels() throws Exception {
+ Path root = new Path("/rootDir");
+ Path appRoot = new Path(root, "appRoot");
+ Path parent = new Path(appRoot, "parentDir");
+ fs.mkdirs(parent);
+ Path child = new Path(parent, "childFile");
+ ContractTestUtils.touch(fs, child);
+
+ Table<String, OmKeyInfo> deletedDirTable =
+ cluster.getOzoneManager().getMetadataManager().getDeletedDirTable();
+ Table<String, OmKeyInfo> keyTable =
+
cluster.getOzoneManager().getMetadataManager().getKeyTable(getFSOBucketLayout());
+ Table<String, OmDirectoryInfo> dirTable =
cluster.getOzoneManager().getMetadataManager().getDirectoryTable();
+
+ DirectoryDeletingService dirDeletingService =
+ (DirectoryDeletingService)
cluster.getOzoneManager().getKeyManager().getDirDeletingService();
+
+ // Before delete
+ assertTableRowCount(deletedDirTable, 0);
+ assertTableRowCount(dirTable, 3);
+ assertTableRowCount(keyTable, 1);
+
+ assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
+ assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
+ assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 0);
+
+ metrics.resetDirectoryMetrics();
+ fs.delete(appRoot, true);
+
+ // After delete
+ checkPath(appRoot);
+ assertTableRowCount(deletedDirTable, 0);
+ assertTableRowCount(keyTable, 0);
+ assertTableRowCount(dirTable, 1);
+ assertSubPathsCount(dirDeletingService::getMovedFilesCount, 1);
+ assertSubPathsCount(dirDeletingService::getDeletedDirsCount, 2);
+ assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
+ assertEquals(2, metrics.getNumDirsSentForPurge());
+ assertEquals(2, metrics.getNumDirsPurged());
+ assertEquals(1, metrics.getNumSubDirsMovedToDeletedDirTable());
+ assertEquals(1, metrics.getNumSubDirsSentForPurge());
+ assertEquals(1, metrics.getNumSubFilesSentForPurge());
+ assertEquals(1, metrics.getNumSubFilesMovedToDeletedTable());
assertThat(dirDeletingService.getRunCount().get()).isGreaterThan(1);
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
new file mode 100644
index 0000000000..5597ff0a6a
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/DeletingServiceMetrics.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.ozone.om;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableGaugeLong;
+import org.apache.hadoop.ozone.OzoneConsts;
+
+/**
+ * Class contains metrics related to the OM Deletion services.
+ */
+@Metrics(about = "Deletion Service Metrics", context = OzoneConsts.OZONE)
+public final class DeletingServiceMetrics {
+
+ public static final String METRICS_SOURCE_NAME =
+ DeletingServiceMetrics.class.getSimpleName();
+ private MetricsRegistry registry;
+
+ private DeletingServiceMetrics() {
+ this.registry = new MetricsRegistry(METRICS_SOURCE_NAME);
+ }
+
+ /**
+ * Creates and returns DeletingServiceMetrics instance.
+ *
+ * @return DeletingServiceMetrics
+ */
+ public static DeletingServiceMetrics create() {
+ return DefaultMetricsSystem.instance().register(METRICS_SOURCE_NAME,
+ "Metrics tracking the progress of deletion of directories and keys in
the OM",
+ new DeletingServiceMetrics());
+ }
+ /**
+ * Unregister the metrics instance.
+ */
+ public static void unregister() {
+ DefaultMetricsSystem.instance().unregisterSource(METRICS_SOURCE_NAME);
+ }
+
+
+ /*
+ * Total directory deletion metrics across all iterations of
DirectoryDeletingService since last restart.
+ */
+ @Metric("Total no. of deleted directories sent for purge")
+ private MutableGaugeLong numDirsSentForPurge;
+ @Metric("Total no. of sub-directories sent for purge")
+ private MutableGaugeLong numSubDirsSentForPurge;
+ @Metric("Total no. of sub-files sent for purge")
+ private MutableGaugeLong numSubFilesSentForPurge;
+
+ public void incrNumDirsSentForPurge(long dirDel) {
+ numDirsSentForPurge.incr(dirDel);
+ }
+
+ public void incrNumSubDirsSentForPurge(long delta) {
+ numSubDirsSentForPurge.incr(delta);
+ }
+
+ public void incrNumSubFilesSentForPurge(long delta) {
+ numSubFilesSentForPurge.incr(delta);
+ }
+
+ public void incrementDirectoryDeletionTotalMetrics(long dirDel, long
dirMove, long filesMove) {
+ incrNumDirsSentForPurge(dirDel);
+ incrNumSubDirsSentForPurge(dirMove);
+ incrNumSubFilesSentForPurge(filesMove);
+ }
+
+ public long getNumDirsSentForPurge() {
+ return numDirsSentForPurge.value();
+ }
+ public long getNumSubDirsSentForPurge() {
+ return numSubDirsSentForPurge.value();
+ }
+ public long getNumSubFilesSentForPurge() {
+ return numSubFilesSentForPurge.value();
+ }
+
+ /*
+ * Total key deletion metrics across all iterations of KeyDeletingService
since last restart.
+ */
+ @Metric("Total no. of keys processed")
+ private MutableGaugeLong numKeysProcessed;
+ @Metric("Total no. of deleted keys sent for purge")
+ private MutableGaugeLong numKeysSentForPurge;
+
+ public void incrNumKeysProcessed(long keysProcessed) {
+ this.numKeysProcessed.incr(keysProcessed);
+ }
+
+ public void incrNumKeysSentForPurge(long keysPurge) {
+ this.numKeysSentForPurge.incr(keysPurge);
+ }
+
+ /*
+ * Directory purge request metrics.
+ */
+ @Metric("Total no. of directories purged")
+ private MutableGaugeLong numDirsPurged;
+ @Metric("Total no. of subFiles moved to deletedTable")
+ private MutableGaugeLong numSubFilesMovedToDeletedTable;
+ @Metric("Total no. of subDirectories moved to deletedDirTable")
+ private MutableGaugeLong numSubDirsMovedToDeletedDirTable;
+
+ public void incrNumDirPurged(long dirPurged) {
+ this.numDirsPurged.incr(dirPurged);
+ }
+ public void incrNumSubFilesMoved(long subKeys) {
+ this.numSubFilesMovedToDeletedTable.incr(subKeys);
+ }
+ public void incrNumSubDirectoriesMoved(long subDirectories) {
+ this.numSubDirsMovedToDeletedDirTable.incr(subDirectories);
+ }
+ public long getNumDirsPurged() {
+ return numDirsPurged.value();
+ }
+ public long getNumSubFilesMovedToDeletedTable() {
+ return numSubFilesMovedToDeletedTable.value();
+ }
+ public long getNumSubDirsMovedToDeletedDirTable() {
+ return numSubDirsMovedToDeletedDirTable.value();
+ }
+
+ /*
+ * Key purge request metrics.
+ */
+ @Metric("Total no. of keys purged")
+ private MutableGaugeLong numKeysPurged;
+
+ public void incrNumKeysPurged(long keysPurged) {
+ this.numKeysPurged.incr(keysPurged);
+ }
+
+ @VisibleForTesting
+ public void resetDirectoryMetrics() {
+ numDirsPurged.set(0);
+ numSubFilesMovedToDeletedTable.set(0);
+ numSubDirsMovedToDeletedDirTable.set(0);
+ numDirsSentForPurge.set(0);
+ numSubDirsSentForPurge.set(0);
+ numSubFilesSentForPurge.set(0);
+ }
+
+}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index d26546e47e..f1d31d130c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -402,6 +402,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
private OMHAMetrics omhaMetrics;
private final ProtocolMessageMetrics<ProtocolMessageEnum>
omClientProtocolMetrics;
+ private final DeletingServiceMetrics omDeletionMetrics;
private OzoneManagerHttpServer httpServer;
private final OMStorage omStorage;
private ObjectName omInfoBeanName;
@@ -684,6 +685,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
metrics = OMMetrics.create();
perfMetrics = OMPerformanceMetrics.register();
+ omDeletionMetrics = DeletingServiceMetrics.create();
// Get admin list
omStarterUser = UserGroupInformation.getCurrentUser().getShortUserName();
omAdmins = OzoneAdmins.getOzoneAdmins(omStarterUser, conf);
@@ -1652,6 +1654,9 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
public OMPerformanceMetrics getPerfMetrics() {
return perfMetrics;
}
+ public DeletingServiceMetrics getDeletionMetrics() {
+ return omDeletionMetrics;
+ }
/**
* Start service.
@@ -2317,6 +2322,7 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
if (omRatisSnapshotProvider != null) {
omRatisSnapshotProvider.close();
}
+ DeletingServiceMetrics.unregister();
OMPerformanceMetrics.unregister();
RatisDropwizardExports.clear(ratisMetricsMap, ratisReporterList);
scmClient.close();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
index b24253e6f6..4758b8cb78 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMDirectoriesPurgeRequestWithFSO.java
@@ -33,7 +33,9 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
+
import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
@@ -75,6 +77,7 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
OmMetadataManagerImpl omMetadataManager = (OmMetadataManagerImpl)
ozoneManager.getMetadataManager();
Map<String, OmKeyInfo> openKeyInfoMap = new HashMap<>();
OMMetrics omMetrics = ozoneManager.getMetrics();
+ DeletingServiceMetrics deletingServiceMetrics =
ozoneManager.getDeletionMetrics();
OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder(
getOmRequest());
final SnapshotInfo fromSnapshotInfo;
@@ -98,6 +101,7 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
return new
OMDirectoriesPurgeResponseWithFSO(createErrorOMResponse(omResponse, e));
}
try {
+ int numSubDirMoved = 0, numSubFilesMoved = 0, numDirsDeleted = 0;
for (OzoneManagerProtocolProtos.PurgePathRequest path : purgeRequests) {
for (OzoneManagerProtocolProtos.KeyInfo key :
path.getMarkDeletedSubDirsList()) {
@@ -111,6 +115,7 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
lockSet.add(volBucketPair);
}
omMetrics.decNumKeys();
+ numSubDirMoved++;
OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager,
volumeName, bucketName);
// bucketInfo can be null in case of delete volume or bucket
@@ -153,6 +158,7 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
}
omMetrics.decNumKeys();
+ numSubFilesMoved++;
OmBucketInfo omBucketInfo = getBucketInfo(omMetadataManager,
volumeName, bucketName);
// bucketInfo can be null in case of delete volume or bucket
@@ -168,7 +174,14 @@ public class OMDirectoriesPurgeRequestWithFSO extends
OMKeyRequest {
volBucketInfoMap.putIfAbsent(volBucketPair, omBucketInfo);
}
}
+ if (path.hasDeletedDir()) {
+ numDirsDeleted++;
+ }
}
+ deletingServiceMetrics.incrNumSubDirectoriesMoved(numSubDirMoved);
+ deletingServiceMetrics.incrNumSubFilesMoved(numSubFilesMoved);
+ deletingServiceMetrics.incrNumDirPurged(numDirsDeleted);
+
if (fromSnapshotInfo != null) {
fromSnapshotInfo.setLastTransactionInfo(TransactionInfo.valueOf(context.getTermIndex()).toByteString());
omMetadataManager.getSnapshotInfoTable().addCacheEntry(new
CacheKey<>(fromSnapshotInfo.getTableKey()),
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
index 5d0af563bb..b77381b674 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyPurgeRequest.java
@@ -26,6 +26,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.exceptions.OMException;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -93,9 +94,14 @@ public class OMKeyPurgeRequest extends OMKeyRequest {
List<String> keysToBePurgedList = new ArrayList<>();
+ int numKeysDeleted = 0;
for (DeletedKeys bucketWithDeleteKeys : bucketDeletedKeysList) {
- keysToBePurgedList.addAll(bucketWithDeleteKeys.getKeysList());
+ List<String> keysList = bucketWithDeleteKeys.getKeysList();
+ keysToBePurgedList.addAll(keysList);
+ numKeysDeleted = numKeysDeleted + keysList.size();
}
+ DeletingServiceMetrics deletingServiceMetrics =
ozoneManager.getDeletionMetrics();
+ deletingServiceMetrics.incrNumKeysPurged(numKeysDeleted);
if (keysToBePurgedList.isEmpty()) {
return new OMKeyPurgeResponse(createErrorOMResponse(omResponse,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
index e60180938f..a3d7ccb661 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/AbstractKeyDeletingService.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.ozone.ClientVersion;
import org.apache.hadoop.ozone.common.BlockGroup;
import org.apache.hadoop.ozone.lock.BootstrapStateHandler;
import org.apache.hadoop.ozone.common.DeleteBlockGroupResult;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OMMetadataManager;
@@ -72,6 +73,7 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
implements BootstrapStateHandler {
private final OzoneManager ozoneManager;
+ private final DeletingServiceMetrics metrics;
private final ScmBlockLocationProtocol scmClient;
private final ClientId clientId = ClientId.randomId();
private final AtomicLong deletedDirsCount;
@@ -92,6 +94,7 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
this.movedDirsCount = new AtomicLong(0);
this.movedFilesCount = new AtomicLong(0);
this.runCount = new AtomicLong(0);
+ this.metrics = ozoneManager.getDeletionMetrics();
}
protected int processKeyDeletes(List<BlockGroup> keyBlocksList,
@@ -450,8 +453,9 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
}
if (dirNum != 0 || subDirNum != 0 || subFileNum != 0) {
+ long subdirMoved = subDirNum - subdirDelNum;
deletedDirsCount.addAndGet(dirNum + subdirDelNum);
- movedDirsCount.addAndGet(subDirNum - subdirDelNum);
+ movedDirsCount.addAndGet(subdirMoved);
movedFilesCount.addAndGet(subFileNum);
long timeTakenInIteration = Time.monotonicNow() - startTime;
LOG.info("Number of dirs deleted: {}, Number of sub-dir " +
@@ -461,6 +465,7 @@ public abstract class AbstractKeyDeletingService extends
BackgroundService
" totalRunCount: {}",
dirNum, subdirDelNum, subFileNum, (subDirNum - subdirDelNum), limit,
timeTakenInIteration, rnCnt);
+ metrics.incrementDirectoryDeletionTotalMetrics(dirNum + subdirDelNum,
subDirNum, subFileNum);
}
return remainNum;
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
index 9a4f74eba5..797b2b9bf9 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyDeletingService.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.BlockGroup;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
import org.apache.hadoop.ozone.om.KeyManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshot;
@@ -98,6 +99,7 @@ public class KeyDeletingService extends
AbstractKeyDeletingService {
private AtomicBoolean isRunningOnAOS;
private final boolean deepCleanSnapshots;
private final SnapshotChainManager snapshotChainManager;
+ private DeletingServiceMetrics metrics;
public KeyDeletingService(OzoneManager ozoneManager,
ScmBlockLocationProtocol scmClient,
@@ -121,6 +123,7 @@ public class KeyDeletingService extends
AbstractKeyDeletingService {
this.isRunningOnAOS = new AtomicBoolean(false);
this.deepCleanSnapshots = deepCleanSnapshots;
this.snapshotChainManager =
((OmMetadataManagerImpl)manager.getMetadataManager()).getSnapshotChainManager();
+ this.metrics = ozoneManager.getDeletionMetrics();
}
/**
@@ -222,6 +225,8 @@ public class KeyDeletingService extends
AbstractKeyDeletingService {
getOzoneManager().getKeyManager(),
pendingKeysDeletion.getKeysToModify(), null,
expectedPreviousSnapshotId);
deletedKeyCount.addAndGet(delCount);
+ metrics.incrNumKeysProcessed(keyBlocksList.size());
+ metrics.incrNumKeysSentForPurge(delCount);
}
} catch (IOException e) {
LOG.error("Error while running delete keys background task. Will " +
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index c18e1ee7c3..547cdbe895 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -33,6 +33,7 @@ import
org.apache.hadoop.hdds.scm.container.common.helpers.ExcludeList;
import org.apache.hadoop.hdds.scm.protocol.StorageContainerLocationProtocol;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.om.DeletingServiceMetrics;
import org.apache.hadoop.ozone.om.IOmMetadataReader;
import org.apache.hadoop.ozone.om.OMPerformanceMetrics;
import org.apache.hadoop.ozone.om.OmMetadataReader;
@@ -119,6 +120,7 @@ public class TestOMKeyRequest {
protected ScmBlockLocationProtocol scmBlockLocationProtocol;
protected StorageContainerLocationProtocol scmContainerLocationProtocol;
protected OMPerformanceMetrics metrics;
+ protected DeletingServiceMetrics delMetrics;
protected static final long CONTAINER_ID = 1000L;
protected static final long LOCAL_ID = 100L;
@@ -139,6 +141,7 @@ public class TestOMKeyRequest {
ozoneManager = mock(OzoneManager.class);
omMetrics = OMMetrics.create();
metrics = OMPerformanceMetrics.register();
+ delMetrics = DeletingServiceMetrics.create();
OzoneConfiguration ozoneConfiguration = getOzoneConfiguration();
ozoneConfiguration.set(OMConfigKeys.OZONE_OM_DB_DIRS,
folder.toAbsolutePath().toString());
@@ -150,6 +153,7 @@ public class TestOMKeyRequest {
ozoneManager);
when(ozoneManager.getMetrics()).thenReturn(omMetrics);
when(ozoneManager.getPerfMetrics()).thenReturn(metrics);
+ when(ozoneManager.getDeletionMetrics()).thenReturn(delMetrics);
when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager);
when(ozoneManager.getConfiguration()).thenReturn(ozoneConfiguration);
OMLayoutVersionManager lvm = mock(OMLayoutVersionManager.class);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
