This is an automated email from the ASF dual-hosted git repository.
swamirishi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 53673c564d HDDS-11244. OmPurgeDirectoriesRequest should clean up File
and Directory tables of AOS for deleted snapshot directories (#8509)
53673c564d is described below
commit 53673c564d6b97d8198233837c9d5e64b8f4d6f7
Author: Swaminathan Balachandran <[email protected]>
AuthorDate: Thu May 29 07:04:02 2025 -0400
HDDS-11244. OmPurgeDirectoriesRequest should clean up File and Directory
tables of AOS for deleted snapshot directories (#8509)
---
.../key/OMDirectoriesPurgeResponseWithFSO.java | 35 +++---
.../ozone/om/request/OMRequestTestUtils.java | 27 +++--
.../TestOMDirectoriesPurgeRequestAndResponse.java | 123 ++++++++++++++++++++-
.../ozone/om/request/key/TestOMKeyRequest.java | 20 +++-
4 files changed, 173 insertions(+), 32 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
index 721ce8401c..115f4986ee 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMDirectoriesPurgeResponseWithFSO.java
@@ -94,13 +94,13 @@ public void addToDBBatch(OMMetadataManager metadataManager,
// Init Batch Operation for snapshot db.
try (BatchOperation writeBatch =
fromSnapshotStore.initBatchOperation()) {
- processPaths(fromSnapshot.getMetadataManager(), writeBatch);
+ processPaths(metadataManager, fromSnapshot.getMetadataManager(),
batchOp, writeBatch);
fromSnapshotStore.commitBatchOperation(writeBatch);
}
}
metadataManager.getSnapshotInfoTable().putWithBatch(batchOp,
fromSnapshotInfo.getTableKey(), fromSnapshotInfo);
} else {
- processPaths(metadataManager, batchOp);
+ processPaths(metadataManager, metadataManager, batchOp, batchOp);
}
// update bucket quota in active db
@@ -111,8 +111,9 @@ public void addToDBBatch(OMMetadataManager metadataManager,
}
}
- public void processPaths(OMMetadataManager omMetadataManager,
- BatchOperation batchOperation) throws IOException {
+ public void processPaths(
+ OMMetadataManager keySpaceOmMetadataManager, OMMetadataManager
deletedSpaceOmMetadataManager,
+ BatchOperation keySpaceBatchOperation, BatchOperation
deletedSpaceBatchOperation) throws IOException {
for (OzoneManagerProtocolProtos.PurgePathRequest path : paths) {
final long volumeId = path.getVolumeId();
final long bucketId = path.getBucketId();
@@ -125,14 +126,14 @@ public void processPaths(OMMetadataManager
omMetadataManager,
// Add all sub-directories to deleted directory table.
for (OzoneManagerProtocolProtos.KeyInfo key : markDeletedSubDirsList) {
OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key);
- String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId,
+ String ozoneDbKey = keySpaceOmMetadataManager.getOzonePathKey(volumeId,
bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName());
- String ozoneDeleteKey = omMetadataManager.getOzoneDeletePathKey(
+ String ozoneDeleteKey =
deletedSpaceOmMetadataManager.getOzoneDeletePathKey(
key.getObjectID(), ozoneDbKey);
- omMetadataManager.getDeletedDirTable().putWithBatch(batchOperation,
+
deletedSpaceOmMetadataManager.getDeletedDirTable().putWithBatch(deletedSpaceBatchOperation,
ozoneDeleteKey, keyInfo);
- omMetadataManager.getDirectoryTable().deleteWithBatch(batchOperation,
+
keySpaceOmMetadataManager.getDirectoryTable().deleteWithBatch(keySpaceBatchOperation,
ozoneDbKey);
if (LOG.isDebugEnabled()) {
@@ -143,10 +144,10 @@ public void processPaths(OMMetadataManager
omMetadataManager,
for (OzoneManagerProtocolProtos.KeyInfo key : deletedSubFilesList) {
OmKeyInfo keyInfo = OmKeyInfo.getFromProtobuf(key);
- String ozoneDbKey = omMetadataManager.getOzonePathKey(volumeId,
+ String ozoneDbKey = keySpaceOmMetadataManager.getOzonePathKey(volumeId,
bucketId, keyInfo.getParentObjectID(), keyInfo.getFileName());
- omMetadataManager.getKeyTable(getBucketLayout())
- .deleteWithBatch(batchOperation, ozoneDbKey);
+ keySpaceOmMetadataManager.getKeyTable(getBucketLayout())
+ .deleteWithBatch(keySpaceBatchOperation, ozoneDbKey);
if (LOG.isDebugEnabled()) {
LOG.info("Move keyName:{} to DeletedTable DBKey: {}",
@@ -156,26 +157,26 @@ public void processPaths(OMMetadataManager
omMetadataManager,
RepeatedOmKeyInfo repeatedOmKeyInfo = OmUtils.prepareKeyForDelete(
keyInfo, keyInfo.getUpdateID());
- String deletedKey = omMetadataManager
+ String deletedKey = keySpaceOmMetadataManager
.getOzoneKey(keyInfo.getVolumeName(), keyInfo.getBucketName(),
keyInfo.getKeyName());
- deletedKey = omMetadataManager.getOzoneDeletePathKey(
+ deletedKey = deletedSpaceOmMetadataManager.getOzoneDeletePathKey(
keyInfo.getObjectID(), deletedKey);
- omMetadataManager.getDeletedTable().putWithBatch(batchOperation,
+
deletedSpaceOmMetadataManager.getDeletedTable().putWithBatch(deletedSpaceBatchOperation,
deletedKey, repeatedOmKeyInfo);
}
if (!openKeyInfoMap.isEmpty()) {
for (Map.Entry<String, OmKeyInfo> entry : openKeyInfoMap.entrySet()) {
- omMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch(
- batchOperation, entry.getKey(), entry.getValue());
+
keySpaceOmMetadataManager.getOpenKeyTable(getBucketLayout()).putWithBatch(
+ keySpaceBatchOperation, entry.getKey(), entry.getValue());
}
}
// Delete the visited directory from deleted directory table
if (path.hasDeletedDir()) {
- omMetadataManager.getDeletedDirTable().deleteWithBatch(batchOperation,
+
deletedSpaceOmMetadataManager.getDeletedDirTable().deleteWithBatch(deletedSpaceBatchOperation,
path.getDeletedDir());
if (LOG.isDebugEnabled()) {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
index cfde5ab856..1264a234dd 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/OMRequestTestUtils.java
@@ -17,6 +17,7 @@
package org.apache.hadoop.ozone.om.request;
+import static
org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doCallRealMethod;
import static org.mockito.Mockito.when;
@@ -503,13 +504,8 @@ public static void addKeyLocationInfo(
*
* @throws Exception
*/
- public static void addDirKeyToDirTable(boolean addToCache,
- OmDirectoryInfo omDirInfo,
- String volume,
- String bucket,
- long trxnLogIndex,
- OMMetadataManager omMetadataManager)
- throws Exception {
+ public static String addDirKeyToDirTable(boolean addToCache, OmDirectoryInfo
omDirInfo, String volume, String bucket,
+ long trxnLogIndex, OMMetadataManager omMetadataManager) throws Exception
{
final long volumeId = omMetadataManager.getVolumeId(volume);
final long bucketId = omMetadataManager.getBucketId(volume, bucket);
final String ozoneKey = omMetadataManager.getOzonePathKey(volumeId,
@@ -520,6 +516,7 @@ public static void addDirKeyToDirTable(boolean addToCache,
CacheValue.get(trxnLogIndex, omDirInfo));
}
omMetadataManager.getDirectoryTable().put(ozoneKey, omDirInfo);
+ return ozoneKey;
}
/**
@@ -1003,6 +1000,22 @@ public static String deleteKey(String ozoneKey,
return ozoneKey;
}
+ /**
+ * Deletes directory from directory table and adds it to DeletedDirectory
table.
+ * @return the deletedDirectoryTable key.
+ */
+ public static String deleteDir(String ozoneKey, String volume, String bucket,
+ OMMetadataManager omMetadataManager)
+ throws IOException {
+ // Retrieve the keyInfo
+ OmDirectoryInfo omDirectoryInfo =
omMetadataManager.getDirectoryTable().get(ozoneKey);
+ OmKeyInfo omKeyInfo = getOmKeyInfo(volume, bucket, omDirectoryInfo,
+ omDirectoryInfo.getName());
+ omMetadataManager.getDeletedDirTable().put(ozoneKey, omKeyInfo);
+ omMetadataManager.getDirectoryTable().delete(ozoneKey);
+ return ozoneKey;
+ }
+
/**
* Create OMRequest which encapsulates InitiateMultipartUpload request.
* @param volumeName
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
index d44b835847..c4682fa220 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMDirectoriesPurgeRequestAndResponse.java
@@ -17,16 +17,22 @@
package org.apache.hadoop.ozone.om.request.key;
+import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
+import static
org.apache.hadoop.ozone.om.request.file.OMFileRequest.getOmKeyInfo;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import jakarta.annotation.Nonnull;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
+import java.util.Random;
import java.util.UUID;
import org.apache.hadoop.hdds.client.BlockID;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.utils.TransactionInfo;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
@@ -36,6 +42,7 @@
import org.apache.hadoop.ozone.om.OmSnapshot;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
@@ -45,8 +52,11 @@
import org.apache.hadoop.ozone.om.response.key.OMKeyPurgeResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.util.Time;
import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
/**
* Tests {@link OMKeyPurgeRequest} and {@link OMKeyPurgeResponse}.
@@ -110,20 +120,24 @@ private void updateBlockInfo(OmKeyInfo omKeyInfo) throws
IOException {
omMetadataManager.getBucketTable().put(bucketKey, omBucketInfo);
}
+ private OMRequest createPurgeKeysRequest(String fromSnapshot, String
purgeDeletedDir,
+ List<OmKeyInfo> keyList, OmBucketInfo bucketInfo) throws IOException {
+ return createPurgeKeysRequest(fromSnapshot, purgeDeletedDir,
Collections.emptyList(), keyList, bucketInfo);
+ }
+
/**
* Create OMRequest which encapsulates DeleteKeyRequest.
* @return OMRequest
*/
private OMRequest createPurgeKeysRequest(String fromSnapshot, String
purgeDeletedDir,
- List<OmKeyInfo> keyList, OmBucketInfo bucketInfo) throws IOException {
+ List<OmKeyInfo> subDirs, List<OmKeyInfo> keyList, OmBucketInfo
bucketInfo) throws IOException {
List<OzoneManagerProtocolProtos.PurgePathRequest> purgePathRequestList
= new ArrayList<>();
List<OmKeyInfo> subFiles = new ArrayList<>();
for (OmKeyInfo key : keyList) {
subFiles.add(key);
}
- List<OmKeyInfo> subDirs = new ArrayList<>();
- Long volumeId = 1L;
+ Long volumeId = omMetadataManager.getVolumeId(bucketInfo.getVolumeName());
Long bucketId = bucketInfo.getObjectID();
OzoneManagerProtocolProtos.PurgePathRequest request = wrapPurgeRequest(
volumeId, bucketId, purgeDeletedDir, subFiles, subDirs);
@@ -184,6 +198,101 @@ private OMRequest preExecute(OMRequest originalOmRequest)
throws IOException {
return modifiedOmRequest;
}
+ @ParameterizedTest
+ @CsvSource(value = {"false,false", "false,true", "true,false", "true,true"})
+ public void testDirectoryPurge(boolean fromSnapshot, boolean purgeDirectory)
throws Exception {
+ Random random = new Random();
+ String bucket = "bucket" + random.nextInt();
+ // Add volume, bucket and key entries to OM DB.
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucket,
+ omMetadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucket);
+ OmBucketInfo bucketInfo =
omMetadataManager.getBucketTable().get(bucketKey);
+ OmDirectoryInfo dir1 = new OmDirectoryInfo.Builder()
+ .setName("dir1")
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .setObjectID(1)
+ .setParentObjectID(bucketInfo.getObjectID())
+ .setUpdateID(0)
+ .build();
+ String dirKey = OMRequestTestUtils.addDirKeyToDirTable(false, dir1,
volumeName, bucket,
+ 1L, omMetadataManager);
+ List<OmKeyInfo> subFiles = new ArrayList<>();
+ List<OmKeyInfo> subDirs = new ArrayList<>();
+ List<String> subFileKeys = new ArrayList<>();
+ List<String> subDirKeys = new ArrayList<>();
+ List<String> deletedSubDirKeys = new ArrayList<>();
+ List<String> deletedSubFiles = new ArrayList<>();
+ for (int id = 1; id < 10; id++) {
+ OmDirectoryInfo subdir = new OmDirectoryInfo.Builder()
+ .setName("subdir" + id)
+ .setCreationTime(Time.now())
+ .setModificationTime(Time.now())
+ .setObjectID(2 * id)
+ .setParentObjectID(dir1.getObjectID())
+ .setUpdateID(0)
+ .build();
+ String subDirectoryPath = OMRequestTestUtils.addDirKeyToDirTable(false,
subdir, volumeName, bucket,
+ 2 * id, omMetadataManager);
+ subDirKeys.add(subDirectoryPath);
+ OmKeyInfo subFile =
+ OMRequestTestUtils.createOmKeyInfo(volumeName, bucket, "file" + id,
RatisReplicationConfig.getInstance(ONE))
+ .setObjectID(2 * id + 1)
+ .setParentObjectID(dir1.getObjectID())
+ .setUpdateID(100L)
+ .build();
+ String subFilePath = OMRequestTestUtils.addFileToKeyTable(false, true,
subFile.getKeyName(),
+ subFile, 1234L, 2 * id + 1, omMetadataManager);
+ subFileKeys.add(subFilePath);
+ subFile.setKeyName("dir1/" + subFile.getKeyName());
+ subFiles.add(subFile);
+ subDirs.add(getOmKeyInfo(volumeName, bucket, subdir,
+ "dir1/" + subdir.getName()));
+
deletedSubDirKeys.add(omMetadataManager.getOzoneDeletePathKey(subdir.getObjectID(),
subDirectoryPath));
+
deletedSubFiles.add(omMetadataManager.getOzoneDeletePathKey(subFile.getObjectID(),
+ omMetadataManager.getOzoneKey(volumeName, bucket,
subFile.getKeyName())));
+ }
+ String deletedDirKey = OMRequestTestUtils.deleteDir(dirKey, volumeName,
bucket, omMetadataManager);
+ for (String subDirKey : subDirKeys) {
+ assertTrue(omMetadataManager.getDirectoryTable().isExist(subDirKey));
+ }
+ for (String subFileKey : subFileKeys) {
+ assertTrue(omMetadataManager.getFileTable().isExist(subFileKey));
+ }
+ assertFalse(omMetadataManager.getDirectoryTable().isExist(dirKey));
+ SnapshotInfo snapshotInfo = null;
+ if (fromSnapshot) {
+ snapshotInfo = createSnapshot(volumeName, bucket, "snapshot");
+ }
+
+ OMRequest omRequest = createPurgeKeysRequest(snapshotInfo == null ? null :
snapshotInfo.getTableKey(),
+ purgeDirectory ? deletedDirKey : null, subDirs, subFiles, bucketInfo);
+ OMRequest preExecutedRequest = preExecute(omRequest);
+ OMDirectoriesPurgeRequestWithFSO omKeyPurgeRequest =
+ new OMDirectoriesPurgeRequestWithFSO(preExecutedRequest);
+ OMDirectoriesPurgeResponseWithFSO omClientResponse =
(OMDirectoriesPurgeResponseWithFSO) omKeyPurgeRequest
+ .validateAndUpdateCache(ozoneManager, 100L);
+ performBatchOperationCommit(omClientResponse);
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshot = fromSnapshot ?
ozoneManager.getOmSnapshotManager()
+ .getSnapshot(snapshotInfo.getSnapshotId()) : null) {
+ OMMetadataManager metadataManager = fromSnapshot ?
snapshot.get().getMetadataManager() :
+ ozoneManager.getMetadataManager();
+ validateDeletedKeys(metadataManager, deletedSubFiles);
+ List<String> deletedDirs = new ArrayList<>(deletedSubDirKeys);
+ if (!purgeDirectory) {
+ deletedDirs.add(deletedDirKey);
+ }
+ validateDeletedDirs(metadataManager, deletedDirs);
+ }
+ for (String subDirKey : subDirKeys) {
+ assertFalse(omMetadataManager.getDirectoryTable().isExist(subDirKey));
+ }
+ for (String subFileKey : subFileKeys) {
+ assertFalse(omMetadataManager.getFileTable().isExist(subFileKey));
+ }
+ }
+
@Test
public void testValidateAndUpdateCacheCheckQuota() throws Exception {
// Create and Delete keys. The keys should be moved to DeletedKeys table
@@ -342,6 +451,14 @@ private List<String>
validateDeletedKeysTable(OMMetadataManager omMetadataManage
return deletedKeyNames;
}
+ private void validateDeletedDirs(OMMetadataManager omMetadataManager,
+ List<String> deletedDirs) throws IOException {
+ for (String deletedDir : deletedDirs) {
+ assertTrue(omMetadataManager.getDeletedDirTable().isExist(
+ deletedDir));
+ }
+ }
+
private void validateDeletedKeys(OMMetadataManager omMetadataManager,
List<String> deletedKeyNames) throws IOException {
for (String deletedKey : deletedKeyNames) {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index b45625b1bd..4354ef8b12 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -248,7 +248,13 @@ public void setup() throws Exception {
.thenReturn(bucket);
when(ozoneManager.resolveBucketLink(any(Pair.class),
any(OMClientRequest.class)))
- .thenReturn(bucket);
+ .thenAnswer(i -> {
+ Pair<String, String> bucketLink = i.getArgument(0);
+ OmBucketInfo bucketInfo = omMetadataManager.getBucketTable().get(
+ omMetadataManager.getBucketKey(bucketLink.getKey(),
bucketLink.getValue()));
+ return new ResolvedBucket(bucketLink.getKey(), bucketLink.getValue(),
+ bucketLink.getKey(), bucketLink.getValue(),
bucketInfo.getOwner(), bucketInfo.getBucketLayout());
+ });
when(ozoneManager.resolveBucketLink(any(Pair.class)))
.thenReturn(bucket);
OmSnapshotManager omSnapshotManager = Mockito.spy(new
OmSnapshotManager(ozoneManager));
@@ -299,15 +305,19 @@ public void stop() {
framework().clearInlineMocks();
}
+ protected SnapshotInfo createSnapshot(String snapshot) throws Exception {
+ return createSnapshot(volumeName, bucketName, snapshot);
+ }
+
/**
* Create snapshot and checkpoint directory.
*/
- protected SnapshotInfo createSnapshot(String snapshotName) throws Exception {
+ protected SnapshotInfo createSnapshot(String volume, String bucket, String
snapshotName) throws Exception {
when(ozoneManager.isAdmin(any())).thenReturn(true);
BatchOperation batchOperation = omMetadataManager.getStore()
.initBatchOperation();
OzoneManagerProtocolProtos.OMRequest omRequest = OMRequestTestUtils
- .createSnapshotRequest(volumeName, bucketName, snapshotName);
+ .createSnapshotRequest(volume, bucket, snapshotName);
// Pre-Execute OMSnapshotCreateRequest.
OMSnapshotCreateRequest omSnapshotCreateRequest =
TestOMSnapshotCreateRequest.doPreExecute(omRequest, ozoneManager);
@@ -320,8 +330,8 @@ protected SnapshotInfo createSnapshot(String snapshotName)
throws Exception {
omMetadataManager.getStore().commitBatchOperation(batchOperation);
batchOperation.close();
- String key = SnapshotInfo.getTableKey(volumeName,
- bucketName, snapshotName);
+ String key = SnapshotInfo.getTableKey(volume,
+ bucket, snapshotName);
SnapshotInfo snapshotInfo =
omMetadataManager.getSnapshotInfoTable().get(key);
assertNotNull(snapshotInfo);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
