This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 3c9e1465e9 HDDS-12350. Reduce duplication between OmBucketReadWrite
tests (#8241)
3c9e1465e9 is described below
commit 3c9e1465e90c107b1d9ec7fe94ffbdd980fa20bf
Author: Priyesh Karatha <[email protected]>
AuthorDate: Thu Apr 17 21:29:55 2025 +0530
HDDS-12350. Reduce duplication between OmBucketReadWrite tests (#8241)
---
.../hadoop/ozone/freon/OmBucketTestUtils.java | 215 +++++++++++++++++++++
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 186 ++----------------
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 60 ++----
3 files changed, 243 insertions(+), 218 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/OmBucketTestUtils.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/OmBucketTestUtils.java
new file mode 100644
index 0000000000..2ff7effa0b
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/OmBucketTestUtils.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.freon;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+import java.util.UUID;
+import org.apache.hadoop.ozone.client.BucketArgs;
+import org.apache.hadoop.ozone.om.lock.OMLockMetrics;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Util Class for OmBucketReadWriteKeyTests.
+ */
+public final class OmBucketTestUtils {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(OmBucketTestUtils.class);
+
+ private OmBucketTestUtils() {
+ }
+
+ static class ParameterBuilder {
+
+ private static final String BUCKET_NAME = "bucket1";
+
+ private final String volumeName = "vol-" + UUID.randomUUID();
+ private final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
+ private String prefixFilePath = "/dir1/dir2";
+ private int countForRead = 100;
+ private int countForWrite = 10;
+ private String dataSize = "256B";
+ private int bufferSize = 64;
+ private int length = 10;
+ private int totalThreadCount = 100;
+ private int readThreadPercentage = 90;
+ private int numOfReadOperations = 50;
+ private int numOfWriteOperations = 10;
+
+ private String description;
+
+ int getExpectedWriteCount() {
+ int readThreadCount = (getReadThreadPercentage() *
getTotalThreadCount()) / 100;
+ int writeThreadCount = getTotalThreadCount() - readThreadCount;
+ return writeThreadCount * getCountForWrite() * getNumOfWriteOperations();
+ }
+
+ ParameterBuilder setPrefixFilePath(String newValue) {
+ prefixFilePath = newValue;
+ return this;
+ }
+
+ ParameterBuilder setCountForRead(int newValue) {
+ countForRead = newValue;
+ return this;
+ }
+
+ ParameterBuilder setCountForWrite(int newValue) {
+ countForWrite = newValue;
+ return this;
+ }
+
+ ParameterBuilder setDataSize(String newValue) {
+ dataSize = newValue;
+ return this;
+ }
+
+ ParameterBuilder setBufferSize(int newValue) {
+ bufferSize = newValue;
+ return this;
+ }
+
+ ParameterBuilder setLength(int newValue) {
+ length = newValue;
+ return this;
+ }
+
+ ParameterBuilder setTotalThreadCount(int newValue) {
+ totalThreadCount = newValue;
+ return this;
+ }
+
+ ParameterBuilder setReadThreadPercentage(int newValue) {
+ readThreadPercentage = newValue;
+ return this;
+ }
+
+ ParameterBuilder setNumOfReadOperations(int newValue) {
+ numOfReadOperations = newValue;
+ return this;
+ }
+
+ ParameterBuilder setNumOfWriteOperations(int newValue) {
+ numOfWriteOperations = newValue;
+ return this;
+ }
+
+ ParameterBuilder setDescription(String newValue) {
+ description = newValue;
+ return this;
+ }
+
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ public String getBucketName() {
+ return BUCKET_NAME;
+ }
+
+ public String getPrefixFilePath() {
+ return prefixFilePath;
+ }
+
+ public BucketArgs.Builder getBucketArgs() {
+ return bucketArgs;
+ }
+
+ public int getBufferSize() {
+ return bufferSize;
+ }
+
+ public String getDataSize() {
+ return dataSize;
+ }
+
+ public int getCountForRead() {
+ return countForRead;
+ }
+
+ public int getCountForWrite() {
+ return countForWrite;
+ }
+
+ public int getLength() {
+ return length;
+ }
+
+ public int getTotalThreadCount() {
+ return totalThreadCount;
+ }
+
+ public int getReadThreadPercentage() {
+ return readThreadPercentage;
+ }
+
+ public int getNumOfReadOperations() {
+ return numOfReadOperations;
+ }
+
+ public int getNumOfWriteOperations() {
+ return numOfWriteOperations;
+ }
+
+ public String getDescription() {
+ return description;
+ }
+
+ @Override
+ public String toString() {
+ return String.format("%s", description);
+ }
+ }
+
+ public static void verifyOMLockMetrics(OMLockMetrics omLockMetrics) {
+ String readLockWaitingTimeMsStat =
+ omLockMetrics.getReadLockWaitingTimeMsStat();
+ LOG.info("Read Lock Waiting Time Stat: " + readLockWaitingTimeMsStat);
+ LOG.info("Longest Read Lock Waiting Time (ms): " +
+ omLockMetrics.getLongestReadLockWaitingTimeMs());
+ int readWaitingSamples =
+ Integer.parseInt(readLockWaitingTimeMsStat.split(" ")[2]);
+ assertThat(readWaitingSamples).isGreaterThan(0);
+
+ String readLockHeldTimeMsStat = omLockMetrics.getReadLockHeldTimeMsStat();
+ LOG.info("Read Lock Held Time Stat: " + readLockHeldTimeMsStat);
+ LOG.info("Longest Read Lock Held Time (ms): " +
+ omLockMetrics.getLongestReadLockHeldTimeMs());
+ int readHeldSamples =
+ Integer.parseInt(readLockHeldTimeMsStat.split(" ")[2]);
+ assertThat(readHeldSamples).isGreaterThan(0);
+
+ String writeLockWaitingTimeMsStat =
+ omLockMetrics.getWriteLockWaitingTimeMsStat();
+ LOG.info("Write Lock Waiting Time Stat: " + writeLockWaitingTimeMsStat);
+ LOG.info("Longest Write Lock Waiting Time (ms): " +
+ omLockMetrics.getLongestWriteLockWaitingTimeMs());
+ int writeWaitingSamples =
+ Integer.parseInt(writeLockWaitingTimeMsStat.split(" ")[2]);
+ assertThat(writeWaitingSamples).isGreaterThan(0);
+
+ String writeLockHeldTimeMsStat =
omLockMetrics.getWriteLockHeldTimeMsStat();
+ LOG.info("Write Lock Held Time Stat: " + writeLockHeldTimeMsStat);
+ LOG.info("Longest Write Lock Held Time (ms): " +
+ omLockMetrics.getLongestWriteLockHeldTimeMs());
+ int writeHeldSamples =
+ Integer.parseInt(writeLockHeldTimeMsStat.split(" ")[2]);
+ assertThat(writeHeldSamples).isGreaterThan(0);
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
index 369f577ce4..38c7766440 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
@@ -17,22 +17,20 @@
package org.apache.hadoop.ozone.freon;
+import static
org.apache.hadoop.ozone.freon.OmBucketTestUtils.verifyOMLockMetrics;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.net.URI;
import java.util.Arrays;
import java.util.List;
-import java.util.UUID;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
-import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.om.lock.OMLockMetrics;
+import org.apache.hadoop.ozone.freon.OmBucketTestUtils.ParameterBuilder;
import org.apache.ozone.test.NonHATests;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.MethodSource;
@@ -54,7 +52,8 @@ static List<ParameterBuilder> parameters() {
.setNumOfWriteOperations(5)
.setReadThreadPercentage(70)
.setCountForRead(10)
- .setCountForWrite(5),
+ .setCountForWrite(5)
+ .setDescription("default"),
new ParameterBuilder()
.setPrefixFilePath("/dir1/dir2/dir3")
.setTotalThreadCount(10)
@@ -63,7 +62,8 @@ static List<ParameterBuilder> parameters() {
.setReadThreadPercentage(80)
.setBufferSize(128)
.setCountForRead(10)
- .setCountForWrite(5),
+ .setCountForWrite(5)
+ .setDescription("with increased buffer and read thread %"),
new ParameterBuilder()
.setPrefixFilePath("/")
.setTotalThreadCount(15)
@@ -71,7 +71,8 @@ static List<ParameterBuilder> parameters() {
.setNumOfWriteOperations(3)
.setDataSize("128B")
.setCountForRead(5)
- .setCountForWrite(3),
+ .setCountForWrite(3)
+ .setDescription("with 128 byte object"),
new ParameterBuilder()
.setPrefixFilePath("/dir1/")
.setTotalThreadCount(10)
@@ -80,13 +81,15 @@ static List<ParameterBuilder> parameters() {
.setCountForRead(5)
.setCountForWrite(3)
.setDataSize("64B")
- .setBufferSize(16),
+ .setBufferSize(16)
+ .setDescription("with 64 byte object"),
new ParameterBuilder()
.setPrefixFilePath("/dir1/dir2/dir3")
.setTotalThreadCount(10)
.setNumOfReadOperations(5)
.setNumOfWriteOperations(0)
- .setCountForRead(5),
+ .setCountForRead(5)
+ .setDescription("pure reads"),
new ParameterBuilder()
.setLength(64)
.setPrefixFilePath("/dir1/dir2/dir3/dir4")
@@ -95,10 +98,11 @@ static List<ParameterBuilder> parameters() {
.setNumOfWriteOperations(5)
.setCountForRead(0)
.setCountForWrite(5)
+ .setDescription("pure writes")
);
}
- @ParameterizedTest
+ @ParameterizedTest(name = "{0}")
@MethodSource("parameters")
void testOmBucketReadWriteFileOps(ParameterBuilder parameterBuilder) throws
Exception {
try (OzoneClient client = cluster().newClient()) {
@@ -165,166 +169,4 @@ private void verifyFileCreation(int expectedCount,
FileStatus[] fileStatuses,
}
assertEquals(expectedCount, actual, "Mismatch Count!");
}
-
- private void verifyOMLockMetrics(OMLockMetrics omLockMetrics) {
- String readLockWaitingTimeMsStat =
- omLockMetrics.getReadLockWaitingTimeMsStat();
- LOG.info("Read Lock Waiting Time Stat: " + readLockWaitingTimeMsStat);
- LOG.info("Longest Read Lock Waiting Time (ms): " +
- omLockMetrics.getLongestReadLockWaitingTimeMs());
- int readWaitingSamples =
- Integer.parseInt(readLockWaitingTimeMsStat.split(" ")[2]);
- assertThat(readWaitingSamples).isPositive();
-
- String readLockHeldTimeMsStat = omLockMetrics.getReadLockHeldTimeMsStat();
- LOG.info("Read Lock Held Time Stat: " + readLockHeldTimeMsStat);
- LOG.info("Longest Read Lock Held Time (ms): " +
- omLockMetrics.getLongestReadLockHeldTimeMs());
- int readHeldSamples =
- Integer.parseInt(readLockHeldTimeMsStat.split(" ")[2]);
- assertThat(readHeldSamples).isPositive();
-
- String writeLockWaitingTimeMsStat =
- omLockMetrics.getWriteLockWaitingTimeMsStat();
- LOG.info("Write Lock Waiting Time Stat: " + writeLockWaitingTimeMsStat);
- LOG.info("Longest Write Lock Waiting Time (ms): " +
- omLockMetrics.getLongestWriteLockWaitingTimeMs());
- int writeWaitingSamples =
- Integer.parseInt(writeLockWaitingTimeMsStat.split(" ")[2]);
- assertThat(writeWaitingSamples).isPositive();
-
- String writeLockHeldTimeMsStat =
omLockMetrics.getWriteLockHeldTimeMsStat();
- LOG.info("Write Lock Held Time Stat: " + writeLockHeldTimeMsStat);
- LOG.info("Longest Write Lock Held Time (ms): " +
- omLockMetrics.getLongestWriteLockHeldTimeMs());
- int writeHeldSamples =
- Integer.parseInt(writeLockHeldTimeMsStat.split(" ")[2]);
- assertThat(writeHeldSamples).isPositive();
- }
-
- static class ParameterBuilder {
-
- private static final String BUCKET_NAME = "bucket1";
-
- private final String volumeName = "vol-" + UUID.randomUUID();
- private final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
- private String prefixFilePath = "/dir1/dir2";
- private int countForRead = 100;
- private int countForWrite = 10;
- private String dataSize = "256B";
- private int bufferSize = 64;
- private int length = 10;
- private int totalThreadCount = 100;
- private int readThreadPercentage = 90;
- private int numOfReadOperations = 50;
- private int numOfWriteOperations = 10;
-
- int getExpectedWriteCount() {
- int readThreadCount = (getReadThreadPercentage() *
getTotalThreadCount()) / 100;
- int writeThreadCount = getTotalThreadCount() - readThreadCount;
- return writeThreadCount * getCountForWrite() * getNumOfWriteOperations();
- }
-
- ParameterBuilder setPrefixFilePath(String newValue) {
- prefixFilePath = newValue;
- return this;
- }
-
- ParameterBuilder setCountForRead(int newValue) {
- countForRead = newValue;
- return this;
- }
-
- ParameterBuilder setCountForWrite(int newValue) {
- countForWrite = newValue;
- return this;
- }
-
- ParameterBuilder setDataSize(String newValue) {
- dataSize = newValue;
- return this;
- }
-
- ParameterBuilder setBufferSize(int newValue) {
- bufferSize = newValue;
- return this;
- }
-
- ParameterBuilder setLength(int newValue) {
- length = newValue;
- return this;
- }
-
- ParameterBuilder setTotalThreadCount(int newValue) {
- totalThreadCount = newValue;
- return this;
- }
-
- ParameterBuilder setReadThreadPercentage(int newValue) {
- readThreadPercentage = newValue;
- return this;
- }
-
- ParameterBuilder setNumOfReadOperations(int newValue) {
- numOfReadOperations = newValue;
- return this;
- }
-
- ParameterBuilder setNumOfWriteOperations(int newValue) {
- numOfWriteOperations = newValue;
- return this;
- }
-
- public String getVolumeName() {
- return volumeName;
- }
-
- public String getBucketName() {
- return BUCKET_NAME;
- }
-
- public String getPrefixFilePath() {
- return prefixFilePath;
- }
-
- public BucketArgs.Builder getBucketArgs() {
- return bucketArgs;
- }
-
- public int getBufferSize() {
- return bufferSize;
- }
-
- public String getDataSize() {
- return dataSize;
- }
-
- public int getCountForRead() {
- return countForRead;
- }
-
- public int getCountForWrite() {
- return countForWrite;
- }
-
- public int getLength() {
- return length;
- }
-
- public int getTotalThreadCount() {
- return totalThreadCount;
- }
-
- public int getReadThreadPercentage() {
- return readThreadPercentage;
- }
-
- public int getNumOfReadOperations() {
- return numOfReadOperations;
- }
-
- public int getNumOfWriteOperations() {
- return numOfWriteOperations;
- }
- }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
index a837cb7851..dc6c3438d4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
@@ -17,8 +17,8 @@
package org.apache.hadoop.ozone.freon;
+import static
org.apache.hadoop.ozone.freon.OmBucketTestUtils.verifyOMLockMetrics;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
-import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
@@ -30,9 +30,8 @@
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneKey;
-import
org.apache.hadoop.ozone.freon.TestOmBucketReadWriteFileOps.ParameterBuilder;
+import org.apache.hadoop.ozone.freon.OmBucketTestUtils.ParameterBuilder;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.apache.hadoop.ozone.om.lock.OMLockMetrics;
import org.apache.hadoop.util.Time;
import org.apache.ozone.test.NonHATests;
import org.junit.jupiter.api.AfterEach;
@@ -70,7 +69,8 @@ static List<ParameterBuilder> parameters() {
.setNumOfWriteOperations(5)
.setReadThreadPercentage(80)
.setCountForRead(10)
- .setCountForWrite(5),
+ .setCountForWrite(5)
+ .setDescription("default"),
new ParameterBuilder()
.setLength(32)
.setTotalThreadCount(10)
@@ -78,13 +78,15 @@ static List<ParameterBuilder> parameters() {
.setNumOfWriteOperations(5)
.setReadThreadPercentage(70)
.setCountForRead(10)
- .setCountForWrite(5),
+ .setCountForWrite(5)
+ .setDescription("with increased length of 32"),
new ParameterBuilder()
.setTotalThreadCount(15)
.setNumOfReadOperations(5)
.setNumOfWriteOperations(3)
.setCountForRead(5)
- .setCountForWrite(3),
+ .setCountForWrite(3)
+ .setDescription("with 0 byte objects"),
new ParameterBuilder()
.setTotalThreadCount(10)
.setNumOfReadOperations(5)
@@ -92,22 +94,25 @@ static List<ParameterBuilder> parameters() {
.setCountForRead(5)
.setCountForWrite(3)
.setDataSize("64B")
- .setBufferSize(16),
+ .setBufferSize(16)
+ .setDescription("with 64 byte object"),
new ParameterBuilder()
.setTotalThreadCount(10)
.setNumOfReadOperations(5)
.setNumOfWriteOperations(0)
- .setCountForRead(5),
+ .setCountForRead(5)
+ .setDescription("pure reads"),
new ParameterBuilder()
.setTotalThreadCount(20)
.setNumOfReadOperations(0)
.setNumOfWriteOperations(5)
.setCountForRead(0)
.setCountForWrite(5)
+ .setDescription("pure writes")
);
}
- @ParameterizedTest(name = "Filesystem Paths Enabled: {0}")
+ @ParameterizedTest(name = "{0}")
@MethodSource("parameters")
void testOmBucketReadWriteKeyOps(ParameterBuilder parameterBuilder) throws
Exception {
OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client,
@@ -154,41 +159,4 @@ private void verifyKeyCreation(int expectedCount,
OzoneBucket bucket,
}
assertEquals(expectedCount, actual, "Mismatch Count!");
}
-
- private void verifyOMLockMetrics(OMLockMetrics omLockMetrics) {
- String readLockWaitingTimeMsStat =
- omLockMetrics.getReadLockWaitingTimeMsStat();
- LOG.info("Read Lock Waiting Time Stat: " + readLockWaitingTimeMsStat);
- LOG.info("Longest Read Lock Waiting Time (ms): " +
- omLockMetrics.getLongestReadLockWaitingTimeMs());
- int readWaitingSamples =
- Integer.parseInt(readLockWaitingTimeMsStat.split(" ")[2]);
- assertThat(readWaitingSamples).isGreaterThan(0);
-
- String readLockHeldTimeMsStat = omLockMetrics.getReadLockHeldTimeMsStat();
- LOG.info("Read Lock Held Time Stat: " + readLockHeldTimeMsStat);
- LOG.info("Longest Read Lock Held Time (ms): " +
- omLockMetrics.getLongestReadLockHeldTimeMs());
- int readHeldSamples =
- Integer.parseInt(readLockHeldTimeMsStat.split(" ")[2]);
- assertThat(readHeldSamples).isGreaterThan(0);
-
- String writeLockWaitingTimeMsStat =
- omLockMetrics.getWriteLockWaitingTimeMsStat();
- LOG.info("Write Lock Waiting Time Stat: " + writeLockWaitingTimeMsStat);
- LOG.info("Longest Write Lock Waiting Time (ms): " +
- omLockMetrics.getLongestWriteLockWaitingTimeMs());
- int writeWaitingSamples =
- Integer.parseInt(writeLockWaitingTimeMsStat.split(" ")[2]);
- assertThat(writeWaitingSamples).isGreaterThan(0);
-
- String writeLockHeldTimeMsStat =
omLockMetrics.getWriteLockHeldTimeMsStat();
- LOG.info("Write Lock Held Time Stat: " + writeLockHeldTimeMsStat);
- LOG.info("Longest Write Lock Held Time (ms): " +
- omLockMetrics.getLongestWriteLockHeldTimeMs());
- int writeHeldSamples =
- Integer.parseInt(writeLockHeldTimeMsStat.split(" ")[2]);
- assertThat(writeHeldSamples).isGreaterThan(0);
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
