This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0781cbede3 HDDS-12694. Disable TestMiniChaosOzoneCluster after fixing
init and shutdown (#8159)
0781cbede3 is described below
commit 0781cbede3786cb0f552d563f98561381a67e06e
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Wed Mar 26 13:36:53 2025 +0100
HDDS-12694. Disable TestMiniChaosOzoneCluster after fixing init and
shutdown (#8159)
---
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 8 +--
.../hadoop/ozone/TestAllMiniChaosOzoneCluster.java | 23 +++----
.../ozone/TestDatanodeMiniChaosOzoneCluster.java | 12 +++-
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 77 +++++++++++-----------
.../TestOzoneManagerMiniChaosOzoneCluster.java | 11 +++-
...orageContainerManagerMiniChaosOzoneCluster.java | 11 +++-
6 files changed, 81 insertions(+), 61 deletions(-)
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
index 6db7b4b016..5871e9b388 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
@@ -132,12 +132,12 @@ void startChaos(long initialDelay, long period, TimeUnit
timeUnit) {
public void shutdown() {
try {
failureManager.stop();
- //this should be called after stopChaos to be sure that the
- //datanode collection is not modified during the shutdown
- super.shutdown();
} catch (Exception e) {
- LOG.error("failed to shutdown MiniOzoneChaosCluster", e);
+ LOG.error("failed to stop FailureManager", e);
}
+ //this should be called after failureManager.stop to be sure that the
+ //datanode collection is not modified during the shutdown
+ super.shutdown();
}
/**
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java
index 086a621d2b..4275beed02 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestAllMiniChaosOzoneCluster.java
@@ -21,33 +21,34 @@
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
import org.apache.hadoop.ozone.failure.Failures;
import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
import picocli.CommandLine;
/**
- * Command line utility to parse and dump a datanode ratis segment file.
+ * Test all kinds of chaos.
*/
@CommandLine.Command(
name = "all",
description = "run chaos cluster across all daemons",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class TestAllMiniChaosOzoneCluster extends TestMiniChaosOzoneCluster
implements Callable<Void> {
- @CommandLine.ParentCommand
- private OzoneChaosCluster chaosCluster;
-
- @Override
- public Void call() throws Exception {
+ @BeforeAll
+ void setup() {
setNumManagers(3, 3, true);
- LoadGenerator.getClassList().forEach(
- TestMiniChaosOzoneCluster::addLoadClasses);
- Failures.getClassList().forEach(
- TestMiniChaosOzoneCluster::addFailureClasses);
+ LoadGenerator.getClassList().forEach(this::addLoadClasses);
+ Failures.getClassList().forEach(this::addFailureClasses);
+ }
+ @Override
+ public Void call() throws Exception {
+ setup();
startChaosCluster();
-
return null;
}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java
index 8a803c2f78..828d57e3db 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestDatanodeMiniChaosOzoneCluster.java
@@ -22,6 +22,8 @@
import org.apache.hadoop.ozone.failure.Failures;
import org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.RandomLoadGenerator;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
import picocli.CommandLine;
/**
@@ -32,19 +34,23 @@
description = "run chaos cluster across Ozone Datanodes",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class TestDatanodeMiniChaosOzoneCluster extends
TestMiniChaosOzoneCluster implements Callable<Void> {
- @Override
- public Void call() throws Exception {
+ @BeforeAll
+ void setup() {
addLoadClasses(RandomLoadGenerator.class);
addLoadClasses(AgedLoadGenerator.class);
addFailureClasses(Failures.DatanodeStartStopFailure.class);
addFailureClasses(Failures.DatanodeRestartFailure.class);
+ }
+ @Override
+ public Void call() throws Exception {
+ setup();
startChaosCluster();
-
return null;
}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
index 53e5b5e042..f06e543353 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
@@ -33,9 +33,9 @@
import org.apache.hadoop.ozone.freon.FreonReplicationOptions;
import org.apache.hadoop.ozone.loadgenerators.LoadGenerator;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
+import org.apache.ozone.test.tag.Unhealthy;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInstance;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
@@ -45,12 +45,14 @@
*/
@Command(description = "Starts IO with MiniOzoneChaosCluster",
name = "chaos", mixinStandardHelpOptions = true)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@Unhealthy("HDDS-3131")
public class TestMiniChaosOzoneCluster extends GenericCli {
- private static List<Class<? extends Failures>> failureClasses
+ private final List<Class<? extends Failures>> failureClasses
= new ArrayList<>();
- private static List<Class<? extends LoadGenerator>> loadClasses
+ private final List<Class<? extends LoadGenerator>> loadClasses
= new ArrayList<>();
enum AllowedBucketLayouts { FILE_SYSTEM_OPTIMIZED, OBJECT_STORE }
@@ -58,66 +60,69 @@ enum AllowedBucketLayouts { FILE_SYSTEM_OPTIMIZED,
OBJECT_STORE }
@Option(names = {"-d", "--num-datanodes", "--numDatanodes"},
description = "num of datanodes. Full name --numDatanodes will be" +
" removed in later versions.")
- private static int numDatanodes = 20;
+ private int numDatanodes = 20;
@Option(names = {"-o", "--num-ozone-manager", "--numOzoneManager"},
description = "num of ozoneManagers. Full name --numOzoneManager will" +
" be removed in later versions.")
- private static int numOzoneManagers = 1;
+ private int numOzoneManagers = 1;
@Option(names = {"-s", "--num-storage-container-manager",
"--numStorageContainerManagers"},
description = "num of storageContainerManagers." +
"Full name --numStorageContainerManagers will" +
" be removed in later versions.")
- private static int numStorageContainerManagerss = 1;
+ private int numStorageContainerManagerss = 1;
@Option(names = {"-t", "--num-threads", "--numThreads"},
description = "num of IO threads. Full name --numThreads will be" +
" removed in later versions.")
- private static int numThreads = 5;
+ private int numThreads = 5;
@Option(names = {"-b", "--num-buffers", "--numBuffers"},
description = "num of IO buffers. Full name --numBuffers will be" +
" removed in later versions.")
- private static int numBuffers = 16;
+ private int numBuffers = 16;
@Option(names = {"-m", "--num-minutes", "--numMinutes"},
description = "total run time. Full name --numMinutes will be " +
"removed in later versions.")
- private static int numMinutes = 1440; // 1 day by default
+ private int numMinutes = 1440; // 1 day by default
@Option(names = {"-v", "--num-data-volume", "--numDataVolume"},
description = "number of datanode volumes to create. Full name " +
"--numDataVolume will be removed in later versions.")
- private static int numDataVolumes = 3;
+ private int numDataVolumes = 3;
+
+ @Option(names = {"--initial-delay"},
+ description = "time (in seconds) before first failure event")
+ private int initialDelay = 300; // seconds
@Option(names = {"-i", "--failure-interval", "--failureInterval"},
description = "time between failure events in seconds. Full name " +
"--failureInterval will be removed in later versions.")
- private static int failureInterval = 300; // 5 minute period between
failures.
+ private int failureInterval = 300; // 5 minute period between failures.
@CommandLine.Mixin
- private static FreonReplicationOptions freonReplication =
+ private FreonReplicationOptions freonReplication =
new FreonReplicationOptions();
@Option(names = {"-l", "--layout"},
description = "Allowed Bucket Layouts: ${COMPLETION-CANDIDATES}")
- private static AllowedBucketLayouts allowedBucketLayout =
+ private AllowedBucketLayouts allowedBucketLayout =
AllowedBucketLayouts.FILE_SYSTEM_OPTIMIZED;
- private static MiniOzoneChaosCluster cluster;
- private static OzoneClient client;
- private static MiniOzoneLoadGenerator loadGenerator;
+ private MiniOzoneChaosCluster cluster;
+ private OzoneClient client;
+ private MiniOzoneLoadGenerator loadGenerator;
- private static String omServiceId = null;
- private static String scmServiceId = null;
+ private String omServiceId;
+ private String scmServiceId;
private static final String OM_SERVICE_ID = "ozoneChaosTest";
private static final String SCM_SERVICE_ID = "scmChaosTest";
- @BeforeAll
- public static void init() throws Exception {
+ private void init() throws Exception {
OzoneConfiguration configuration = new OzoneConfiguration();
MiniOzoneChaosCluster.Builder chaosBuilder =
@@ -164,19 +169,19 @@ public static void init() throws Exception {
loadGenerator = loadBuilder.build();
}
- static void addFailureClasses(Class<? extends Failures> clz) {
+ void addFailureClasses(Class<? extends Failures> clz) {
failureClasses.add(clz);
}
- static void addLoadClasses(Class<? extends LoadGenerator> clz) {
+ void addLoadClasses(Class<? extends LoadGenerator> clz) {
loadClasses.add(clz);
}
- static void setNumDatanodes(int nDns) {
+ void setNumDatanodes(int nDns) {
numDatanodes = nDns;
}
- static void setNumManagers(int nOms, int numScms, boolean enableHA) {
+ void setNumManagers(int nOms, int numScms, boolean enableHA) {
if (nOms > 1 || enableHA) {
omServiceId = OM_SERVICE_ID;
@@ -189,26 +194,18 @@ static void setNumManagers(int nOms, int numScms, boolean
enableHA) {
numStorageContainerManagerss = numScms;
}
- /**
- * Shutdown MiniDFSCluster.
- */
- @AfterAll
- public static void shutdown() {
+ private void shutdown() {
if (loadGenerator != null) {
loadGenerator.shutdownLoadGenerator();
}
- IOUtils.closeQuietly(client);
-
- if (cluster != null) {
- cluster.shutdown();
- }
+ IOUtils.closeQuietly(client, cluster);
}
public void startChaosCluster() throws Exception {
try {
init();
- cluster.startChaos(failureInterval, failureInterval, TimeUnit.SECONDS);
+ cluster.startChaos(initialDelay, failureInterval, TimeUnit.SECONDS);
loadGenerator.startIO(numMinutes, TimeUnit.MINUTES);
} finally {
shutdown();
@@ -216,8 +213,10 @@ public void startChaosCluster() throws Exception {
}
@Test
- public void testReadWriteWithChaosCluster() throws Exception {
- cluster.startChaos(5, 10, TimeUnit.SECONDS);
- loadGenerator.startIO(120, TimeUnit.SECONDS);
+ void test() throws Exception {
+ initialDelay = 5; // seconds
+ failureInterval = 10; // seconds
+ numMinutes = 2;
+ startChaosCluster();
}
}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java
index e7e024d49a..662e3f4b7c 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestOzoneManagerMiniChaosOzoneCluster.java
@@ -23,6 +23,8 @@
import org.apache.hadoop.ozone.loadgenerators.AgedDirLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.NestedDirLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.RandomDirLoadGenerator;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
import picocli.CommandLine;
/**
@@ -33,11 +35,12 @@
description = "run chaos cluster across Ozone Managers",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class TestOzoneManagerMiniChaosOzoneCluster extends
TestMiniChaosOzoneCluster implements Callable<Void> {
- @Override
- public Void call() throws Exception {
+ @BeforeAll
+ void setup() {
setNumManagers(3, 1, true);
setNumDatanodes(3);
@@ -47,7 +50,11 @@ public Void call() throws Exception {
addFailureClasses(Failures.OzoneManagerRestartFailure.class);
addFailureClasses(Failures.OzoneManagerStartStopFailure.class);
+ }
+ @Override
+ public Void call() throws Exception {
+ setup();
startChaosCluster();
return null;
}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerMiniChaosOzoneCluster.java
index bbaa00130c..bf0ea95663 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerMiniChaosOzoneCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerMiniChaosOzoneCluster.java
@@ -23,6 +23,8 @@
import org.apache.hadoop.ozone.loadgenerators.AgedDirLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.NestedDirLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.RandomDirLoadGenerator;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
import picocli.CommandLine;
/**
@@ -33,11 +35,12 @@
description = "run chaos cluster across Storage Container Managers",
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class)
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public class TestStorageContainerManagerMiniChaosOzoneCluster extends
TestMiniChaosOzoneCluster implements Callable<Void> {
- @Override
- public Void call() throws Exception {
+ @BeforeAll
+ void setup() {
setNumManagers(3, 3, true);
setNumDatanodes(3);
@@ -47,7 +50,11 @@ public Void call() throws Exception {
addFailureClasses(Failures.StorageContainerManagerRestartFailure.class);
addFailureClasses(Failures.StorageContainerManagerStartStopFailure.class);
+ }
+ @Override
+ public Void call() throws Exception {
+ setup();
startChaosCluster();
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
