This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 9a164854f99 HDDS-11314. OM system audit for internal request and
leader change (#9092)
9a164854f99 is described below
commit 9a164854f993b79c5e2bea9839692fac52e7b9f2
Author: Sarveksha Yeshavantha Raju
<[email protected]>
AuthorDate: Mon Oct 13 14:54:59 2025 +0530
HDDS-11314. OM system audit for internal request and leader change (#9092)
---
.../hadoop/ozone/om/TestOMRatisSnapshots.java | 8 +++
.../apache/hadoop/ozone/audit/OMSystemAction.java | 3 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 13 +++++
.../ozone/om/ratis/OzoneManagerStateMachine.java | 21 +++++++
.../om/request/key/OMOpenKeysDeleteRequest.java | 29 +++++++++-
.../request/key/TestOMOpenKeysDeleteRequest.java | 65 ++++++++++++++++++++++
6 files changed, 137 insertions(+), 2 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
index 404a32d2178..a1de8fc377a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOMRatisSnapshots.java
@@ -67,6 +67,7 @@
import org.apache.hadoop.hdds.utils.db.RDBStore;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.MiniOzoneHAClusterImpl;
+import org.apache.hadoop.ozone.audit.AuditLogTestUtils;
import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -123,6 +124,7 @@ public class TestOMRatisSnapshots {
private static final BucketLayout TEST_BUCKET_LAYOUT =
BucketLayout.OBJECT_STORE;
private OzoneClient client;
+ private GenericTestUtils.PrintStreamCapturer output;
/**
* Create a MiniOzoneCluster for testing. The cluster initially has one
@@ -145,6 +147,8 @@ public void init(TestInfo testInfo) throws Exception {
testInfo.getTestMethod().get().getName()
.equals("testInstallSnapshot")) {
snapshotThreshold = SNAPSHOT_THRESHOLD * 10;
+ AuditLogTestUtils.enableAuditLog();
+ output = GenericTestUtils.captureOut();
}
conf.setLong(
OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY,
@@ -285,6 +289,8 @@ public void testInstallSnapshot(@TempDir Path tempDir)
throws Exception {
assertLogCapture(logCapture,
"Install Checkpoint is finished");
+ assertThat(output.get()).contains("op=DB_CHECKPOINT_INSTALL
{\"leaderId\":\"" + leaderOMNodeId + "\",\"term\":\"" +
+ leaderOMSnapshotTermIndex, "\"lastAppliedIndex\":\"" +
followerOMLastAppliedIndex);
// Read & Write after snapshot installed.
List<String> newKeys = writeKeys(1);
@@ -308,6 +314,8 @@ public void testInstallSnapshot(@TempDir Path tempDir)
throws Exception {
// Confirm that there was no overlap of sst files
// between the individual tarballs.
assertEquals(sstFileUnion.size(), sstFileCount);
+
+ output.reset();
}
private void checkSnapshot(OzoneManager leaderOM, OzoneManager followerOM,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java
index 5da25966a28..3fbceeeae9c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/audit/OMSystemAction.java
@@ -23,6 +23,9 @@
*/
public enum OMSystemAction implements AuditAction {
STARTUP,
+ LEADER_CHANGE,
+ OPEN_KEY_CLEANUP,
+ DB_CHECKPOINT_INSTALL,
DIRECTORY_DELETION,
KEY_DELETION,
SNAPSHOT_MOVE_TABLE_KEYS,
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index 75dd29d67bf..56e51cf4026 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -362,6 +362,10 @@ public final class OzoneManager extends
ServiceRuntimeInfoImpl
private static final AuditLogger SYSTEMAUDIT = new AuditLogger(
AuditLoggerType.OMSYSTEMLOGGER);
+ private static final String AUDIT_PARAM_LEADER_ID = "leaderId";
+ private static final String AUDIT_PARAM_TERM = "term";
+ private static final String AUDIT_PARAM_LAST_APPLIED_INDEX =
"lastAppliedIndex";
+
private static final String OM_DAEMON = "om";
private static final String NO_LEADER_ERROR_MESSAGE =
"There is no leader among the Ozone Manager servers. If this message
" +
@@ -4136,6 +4140,7 @@ TermIndex installCheckpoint(String leaderId, Path
checkpointLocation,
exitManager.exitSystem(1, errorMsg, e, LOG);
}
}
+ buildDBCheckpointInstallAuditLog(leaderId, term, lastAppliedIndex);
// Delete the backup DB
try {
@@ -4162,6 +4167,14 @@ TermIndex installCheckpoint(String leaderId, Path
checkpointLocation,
return newTermIndex;
}
+ private void buildDBCheckpointInstallAuditLog(String leaderId, long term,
long lastAppliedIndex) {
+ Map<String, String> auditMap = new LinkedHashMap<>();
+ auditMap.put(AUDIT_PARAM_LEADER_ID, leaderId);
+ auditMap.put(AUDIT_PARAM_TERM, String.valueOf(term));
+ auditMap.put(AUDIT_PARAM_LAST_APPLIED_INDEX,
String.valueOf(lastAppliedIndex));
+
SYSTEMAUDIT.logWriteSuccess(buildAuditMessageForSuccess(OMSystemAction.DB_CHECKPOINT_INSTALL,
auditMap));
+ }
+
private void stopTrashEmptier() {
if (this.emptier != null) {
emptier.interrupt();
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
index a304e39dec3..a3ad217ceef 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/OzoneManagerStateMachine.java
@@ -26,7 +26,9 @@
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.LinkedHashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadFactory;
@@ -34,6 +36,9 @@
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hdds.utils.NettyMetrics;
import org.apache.hadoop.hdds.utils.TransactionInfo;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.audit.OMSystemAction;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.OzoneManagerPrepareState;
@@ -85,6 +90,10 @@ public class OzoneManagerStateMachine extends
BaseStateMachine {
private static final Logger LOG =
LoggerFactory.getLogger(OzoneManagerStateMachine.class);
+ private static final AuditLogger AUDIT = new
AuditLogger(AuditLoggerType.OMSYSTEMLOGGER);
+ private static final String AUDIT_PARAM_PREVIOUS_LEADER = "previousLeader";
+ private static final String AUDIT_PARAM_NEW_LEADER = "newLeader";
+ private RaftPeerId previousLeaderId = null;
private final SimpleStateMachineStorage storage =
new SimpleStateMachineStorage();
private final OzoneManager ozoneManager;
@@ -169,8 +178,20 @@ public void notifyLeaderChanged(RaftGroupMemberId
groupMemberId,
// warmup cache
ozoneManager.initializeEdekCache(ozoneManager.getConfiguration());
}
+ // Store the previous leader before updating
+ RaftPeerId actualPreviousLeader = previousLeaderId;
+
+ // Update the previous leader for next time
+ previousLeaderId = newLeaderId;
// Initialize OMHAMetrics
ozoneManager.omHAMetricsInit(newLeaderId.toString());
+
+ Map<String, String> auditParams = new LinkedHashMap<>();
+ auditParams.put(AUDIT_PARAM_PREVIOUS_LEADER,
+ actualPreviousLeader != null ? String.valueOf(actualPreviousLeader) :
"NONE");
+ auditParams.put(AUDIT_PARAM_NEW_LEADER, String.valueOf(newLeaderId));
+
AUDIT.logWriteSuccess(ozoneManager.buildAuditMessageForSuccess(OMSystemAction.LEADER_CHANGE,
auditParams));
+
LOG.info("{}: leader changed to {}", groupMemberId, newLeaderId);
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
index 45606468477..3997f1a6778 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMOpenKeysDeleteRequest.java
@@ -21,12 +21,17 @@
import java.io.IOException;
import java.nio.file.InvalidPathException;
+import java.util.ArrayList;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
+import org.apache.hadoop.ozone.audit.AuditLogger;
+import org.apache.hadoop.ozone.audit.AuditLoggerType;
+import org.apache.hadoop.ozone.audit.OMSystemAction;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OMMetrics;
import org.apache.hadoop.ozone.om.OzoneManager;
@@ -55,6 +60,10 @@ public class OMOpenKeysDeleteRequest extends OMKeyRequest {
private static final Logger LOG =
LoggerFactory.getLogger(OMOpenKeysDeleteRequest.class);
+ private static final AuditLogger AUDIT = new
AuditLogger(AuditLoggerType.OMSYSTEMLOGGER);
+ private static final String AUDIT_PARAM_NUM_OPEN_KEYS = "numOpenKeysDeleted";
+ private static final String AUDIT_PARAM_OPEN_KEYS = "openKeysDeleted";
+
public OMOpenKeysDeleteRequest(OMRequest omRequest,
BucketLayout bucketLayout) {
super(omRequest, bucketLayout);
@@ -89,7 +98,7 @@ public OMClientResponse validateAndUpdateCache(OzoneManager
ozoneManager, Execut
Result result = null;
// Map containing a pair of BucketId and delete key info.
Map<String, Pair<Long, OmKeyInfo>> deletedOpenKeys = new HashMap<>();
-
+ Map<String, String> auditParams = new LinkedHashMap<>();
try {
for (OpenKeyBucket openKeyBucket: submittedOpenKeyBuckets) {
// For each bucket where keys will be deleted from,
@@ -102,8 +111,23 @@ public OMClientResponse
validateAndUpdateCache(OzoneManager ozoneManager, Execut
deletedOpenKeys, getBucketLayout());
result = Result.SUCCESS;
+
+ List<String> deletedOpenKeysLight = new
ArrayList<>(deletedOpenKeys.size());
+ for (Pair<Long, OmKeyInfo> key : deletedOpenKeys.values()) {
+ OmKeyInfo keyInfo = key.getRight();
+ OzoneManagerProtocolProtos.KeyArgs keyArgs =
OzoneManagerProtocolProtos.KeyArgs.newBuilder()
+ .setVolumeName(keyInfo.getVolumeName())
+ .setBucketName(keyInfo.getBucketName())
+ .setKeyName(keyInfo.getKeyName())
+ .build();
+
deletedOpenKeysLight.add(buildLightKeyArgsAuditMap(keyArgs).toString());
+ }
+ auditParams.put(AUDIT_PARAM_NUM_OPEN_KEYS,
String.valueOf(deletedOpenKeys.size()));
+ auditParams.put(AUDIT_PARAM_OPEN_KEYS, deletedOpenKeysLight.toString());
+
AUDIT.logWriteSuccess(ozoneManager.buildAuditMessageForSuccess(OMSystemAction.OPEN_KEY_CLEANUP,
auditParams));
} catch (IOException | InvalidPathException ex) {
result = Result.FAILURE;
+
AUDIT.logWriteFailure(ozoneManager.buildAuditMessageForFailure(OMSystemAction.OPEN_KEY_CLEANUP,
auditParams, ex));
exception = ex;
omClientResponse =
new OMOpenKeysDeleteResponse(createErrorOMResponse(omResponse,
@@ -140,7 +164,7 @@ private void processResults(OMMetrics omMetrics, long
numSubmittedOpenKeys,
}
}
- private void updateOpenKeyTableCache(OzoneManager ozoneManager,
+ protected void updateOpenKeyTableCache(OzoneManager ozoneManager,
long trxnLogIndex, OpenKeyBucket keysPerBucket,
Map<String, Pair<Long, OmKeyInfo>> deletedOpenKeys) throws IOException {
@@ -148,6 +172,7 @@ private void updateOpenKeyTableCache(OzoneManager
ozoneManager,
String volumeName = keysPerBucket.getVolumeName();
String bucketName = keysPerBucket.getBucketName();
OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager();
+
try {
mergeOmLockDetails(omMetadataManager.getLock()
.acquireWriteLock(BUCKET_LOCK, volumeName, bucketName));
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
index 53f384b65ac..57a7b5a1f9d 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMOpenKeysDeleteRequest.java
@@ -21,19 +21,31 @@
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.anyMap;
+import static org.mockito.Mockito.any;
+import static org.mockito.Mockito.anyLong;
+import static org.mockito.Mockito.argThat;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.eq;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.List;
+import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
+import org.apache.hadoop.ozone.audit.OMSystemAction;
import org.apache.hadoop.ozone.om.OMMetrics;
+import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
@@ -296,6 +308,59 @@ public void testMetrics(BucketLayout buckLayout) throws
Exception {
assertEquals(numExistentKeys, metrics.getNumOpenKeysDeleted());
}
+ /**
+ * Test OPEN_KEY_CLEANUP audit logging for both success and failure cases
when open keys are deleted.
+ */
+ @ParameterizedTest
+ @MethodSource("bucketLayouts")
+ public void testOpenKeyCleanupAuditLogging(BucketLayout buckLayout) throws
Exception {
+ this.bucketLayout = buckLayout;
+ final String volume = UUID.randomUUID().toString();
+ final String bucket = UUID.randomUUID().toString();
+
+ OMRequestTestUtils.addVolumeAndBucketToDB(volume, bucket,
+ omMetadataManager, getBucketLayout());
+
+ List<Pair<Long, OmKeyInfo>> openKeys = makeOpenKeys(volume, bucket, 3);
+ addToOpenKeyTableDB(openKeys);
+
+ OMRequest omRequest = doPreExecute(createDeleteOpenKeyRequest(openKeys));
+ OMOpenKeysDeleteRequest openKeyDeleteRequest = spy(new
OMOpenKeysDeleteRequest(omRequest, getBucketLayout()));
+
+ OMClientResponse omClientResponse =
+ openKeyDeleteRequest.validateAndUpdateCache(ozoneManager, 100L);
+
+ assertEquals(Status.OK, omClientResponse.getOMResponse().getStatus());
+
+ verify(ozoneManager, times(1))
+ .buildAuditMessageForSuccess(eq(OMSystemAction.OPEN_KEY_CLEANUP),
+ argThat(params -> {
+ assertEquals("3", params.get("numOpenKeysDeleted"));
+ assertTrue(params.containsKey("openKeysDeleted"));
+ return true;
+ }));
+
+ assertNotInOpenKeyTable(openKeys);
+
+ // Simulate failure by mocking updateOpenKeyTableCache to throw an
IOException, and verify the failure audit log.
+ doThrow(new IOException())
+ .when(openKeyDeleteRequest)
+ .updateOpenKeyTableCache(
+ any(OzoneManager.class),
+ anyLong(),
+ any(OpenKeyBucket.class),
+ anyMap());
+
+ omClientResponse =
openKeyDeleteRequest.validateAndUpdateCache(ozoneManager, 100L);
+
+ assertEquals(Status.INTERNAL_ERROR,
omClientResponse.getOMResponse().getStatus());
+
+ verify(ozoneManager, times(1))
+ .buildAuditMessageForFailure(eq(OMSystemAction.OPEN_KEY_CLEANUP),
+ argThat(Map::isEmpty),
+ any(Throwable.class));
+ }
+
/**
* Runs the validate and update cache step of
* {@link OMOpenKeysDeleteRequest} to mark the keys in {@code openKeys}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
