This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5733c5556e HDDS-11799. Remove config
hdds.scm.safemode.pipeline-availability.check (#8095)
5733c5556e is described below
commit 5733c5556e745580d1ca31a2c956806c03d76c7e
Author: Peter Lee <[email protected]>
AuthorDate: Tue Apr 8 18:05:08 2025 +0800
HDDS-11799. Remove config hdds.scm.safemode.pipeline-availability.check
(#8095)
---
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 --
.../common/src/main/resources/ozone-default.xml | 9 ---
.../hdds/scm/pipeline/PipelineReportHandler.java | 7 +-
.../hadoop/hdds/scm/safemode/SafeModeMetrics.java | 3 +-
.../hdds/scm/safemode/SafeModeRuleFactory.java | 10 +--
.../safemode/TestHealthyPipelineSafeModeRule.java | 6 --
.../TestOneReplicaPipelineSafeModeRule.java | 2 -
.../hdds/scm/safemode/TestSCMSafeModeManager.java | 84 +++++++++-------------
.../TestSCMPipelineBytesWrittenMetrics.java | 3 -
.../safemode/TestSCMSafeModeWithPipelineRules.java | 3 -
10 files changed, 39 insertions(+), 93 deletions(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
index c76cafda55..20f21ce5ca 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/HddsConfigKeys.java
@@ -89,11 +89,6 @@ public final class HddsConfigKeys {
public static final String
HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT_DEFAULT = "5m";
- public static final String HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK =
- "hdds.scm.safemode.pipeline-availability.check";
- public static final boolean
- HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT = true;
-
public static final String HDDS_SCM_SAFEMODE_PIPELINE_CREATION =
"hdds.scm.safemode.pipeline.creation";
public static final boolean
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index 7e76cf68c7..3bc86c370c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -1659,15 +1659,6 @@
</description>
</property>
- <property>
- <name>hdds.scm.safemode.pipeline-availability.check</name>
- <value>true</value>
- <tag>HDDS,SCM,OPERATION</tag>
- <description>
- Boolean value to enable pipeline availability check during SCM safe mode.
- </description>
- </property>
-
<property>
<name>hdds.scm.safemode.healthy.pipeline.pct</name>
<value>0.10</value>
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
index dbdb366a91..b7c295ef81 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineReportHandler.java
@@ -20,7 +20,6 @@
import com.google.common.base.Preconditions;
import java.io.IOException;
import java.util.concurrent.TimeoutException;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
@@ -52,7 +51,6 @@ public class PipelineReportHandler implements
private final PipelineManager pipelineManager;
private final SafeModeManager scmSafeModeManager;
private final SCMContext scmContext;
- private final boolean pipelineAvailabilityCheck;
private final SCMPipelineMetrics metrics;
public PipelineReportHandler(SafeModeManager scmSafeModeManager,
@@ -64,9 +62,6 @@ public PipelineReportHandler(SafeModeManager
scmSafeModeManager,
this.pipelineManager = pipelineManager;
this.scmContext = scmContext;
this.metrics = SCMPipelineMetrics.create();
- this.pipelineAvailabilityCheck = conf.getBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT);
}
@Override
@@ -140,7 +135,7 @@ protected void processPipelineReport(PipelineReport report,
}
}
if (pipeline.isHealthy()) {
- if (pipelineAvailabilityCheck && scmSafeModeManager.getInSafeMode()) {
+ if (scmSafeModeManager.getInSafeMode()) {
publisher.fireEvent(SCMEvents.OPEN_PIPELINE, pipeline);
}
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeMetrics.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeMetrics.java
index 8099f6db3f..bcdfa81425 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeMetrics.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeMetrics.java
@@ -42,8 +42,7 @@ public class SafeModeMetrics {
private @Metric MutableCounterLong
currentContainersWithECDataReplicaReportedCount;
- // When hdds.scm.safemode.pipeline-availability.check is set then only
- // below metrics will have some values, otherwise they will be zero.
+ // Pipeline metrics for safemode
private @Metric MutableGaugeLong numHealthyPipelinesThreshold;
private @Metric MutableCounterLong currentHealthyPipelinesCount;
private @Metric MutableGaugeLong
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java
index 9d1d732e2e..65be38ae6e 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/safemode/SafeModeRuleFactory.java
@@ -19,7 +19,6 @@
import java.util.ArrayList;
import java.util.List;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.scm.container.ContainerManager;
import org.apache.hadoop.hdds.scm.ha.SCMContext;
@@ -79,13 +78,8 @@ private void loadRules() {
preCheckRules.add(datanodeRule);
- // TODO: Move isRuleEnabled check to the Rule implementation. (HDDS-11799)
- if (config.getBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK_DEFAULT)
- && pipelineManager != null) {
-
- safeModeRules.add(new HealthyPipelineSafeModeRule(eventQueue,
pipelineManager,
+ if (pipelineManager != null) {
+ safeModeRules.add(new HealthyPipelineSafeModeRule(eventQueue,
pipelineManager,
safeModeManager, config, scmContext));
safeModeRules.add(new OneReplicaPipelineSafeModeRule(eventQueue,
pipelineManager,
safeModeManager, config));
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
index c2c384e10d..dde1076a6b 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestHealthyPipelineSafeModeRule.java
@@ -74,8 +74,6 @@ public void testHealthyPipelineSafeModeRuleWithNoPipelines()
when(containerManager.getContainers()).thenReturn(containers);
config.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempFile.getPath());
// enable pipeline check
- config.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
true);
config.setBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
SCMMetadataStore scmMetadataStore = new SCMMetadataStoreImpl(config);
@@ -128,8 +126,6 @@ public void testHealthyPipelineSafeModeRuleWithPipelines()
throws Exception {
when(containerManager.getContainers()).thenReturn(containers);
config.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempFile.getPath());
// enable pipeline check
- config.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
true);
config.setBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
@@ -224,8 +220,6 @@ public void
testHealthyPipelineSafeModeRuleWithMixedPipelines()
when(containerManager.getContainers()).thenReturn(containers);
config.set(HddsConfigKeys.OZONE_METADATA_DIRS, tempFile.getPath());
// enable pipeline check
- config.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
true);
config.setBoolean(
HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
index 341643100b..64bb2d21fb 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestOneReplicaPipelineSafeModeRule.java
@@ -77,8 +77,6 @@ public class TestOneReplicaPipelineSafeModeRule {
private void setup(int nodes, int pipelineFactorThreeCount,
int pipelineFactorOneCount) throws Exception {
OzoneConfiguration ozoneConfiguration = new OzoneConfiguration();
- ozoneConfiguration.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
ozoneConfiguration.set(HddsConfigKeys.OZONE_METADATA_DIRS,
tempDir.toString());
ozoneConfiguration.setBoolean(
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
index 78d0f8d3df..f693df81dc 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeManager.java
@@ -208,9 +208,6 @@ public void testSafeModeExitRule() throws Exception {
private OzoneConfiguration createConf(double healthyPercent,
double oneReplicaPercent) {
OzoneConfiguration conf = new OzoneConfiguration(config);
- conf.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
- true);
conf.setDouble(HddsConfigKeys.
HDDS_SCM_SAFEMODE_HEALTHY_PIPELINE_THRESHOLD_PCT, healthyPercent);
conf.setDouble(HddsConfigKeys.
@@ -667,67 +664,56 @@ private void testECContainerThreshold(List<ContainerInfo>
dnContainers,
public void testSafeModePipelineExitRule() throws Exception {
containers = new ArrayList<>();
containers.addAll(HddsTestUtils.getContainerInfo(25 * 4));
- try {
- MockNodeManager nodeManager = new MockNodeManager(true, 3);
- // enable pipeline check
- config.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
-
- PipelineManagerImpl pipelineManager =
- PipelineManagerImpl.newPipelineManager(
- config,
- SCMHAManagerStub.getInstance(true),
- nodeManager,
- scmMetadataStore.getPipelineTable(),
- queue,
- scmContext,
- serviceManager,
- Clock.system(ZoneOffset.UTC));
-
- PipelineProvider<RatisReplicationConfig> mockRatisProvider =
- new MockRatisPipelineProvider(nodeManager,
- pipelineManager.getStateManager(), config);
- pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
- mockRatisProvider);
+ MockNodeManager nodeManager = new MockNodeManager(true, 3);
+ PipelineManagerImpl pipelineManager =
+ PipelineManagerImpl.newPipelineManager(
+ config,
+ SCMHAManagerStub.getInstance(true),
+ nodeManager,
+ scmMetadataStore.getPipelineTable(),
+ queue,
+ scmContext,
+ serviceManager,
+ Clock.system(ZoneOffset.UTC));
- Pipeline pipeline = pipelineManager.createPipeline(
- RatisReplicationConfig.getInstance(
- ReplicationFactor.THREE));
+ PipelineProvider<RatisReplicationConfig> mockRatisProvider =
+ new MockRatisPipelineProvider(nodeManager,
+ pipelineManager.getStateManager(), config);
+ pipelineManager.setPipelineProvider(HddsProtos.ReplicationType.RATIS,
+ mockRatisProvider);
- pipeline = pipelineManager.getPipeline(pipeline.getId());
- MockRatisPipelineProvider.markPipelineHealthy(pipeline);
- ContainerManager containerManager = mock(ContainerManager.class);
- when(containerManager.getContainers()).thenReturn(containers);
+ Pipeline pipeline = pipelineManager.createPipeline(
+ RatisReplicationConfig.getInstance(
+ ReplicationFactor.THREE));
+
+ pipeline = pipelineManager.getPipeline(pipeline.getId());
+ MockRatisPipelineProvider.markPipelineHealthy(pipeline);
+ ContainerManager containerManager = mock(ContainerManager.class);
+ when(containerManager.getContainers()).thenReturn(containers);
- scmSafeModeManager = new SCMSafeModeManager(
+ scmSafeModeManager = new SCMSafeModeManager(
config, containerManager, pipelineManager, nodeManager, queue,
serviceManager, scmContext);
- SCMDatanodeProtocolServer.NodeRegistrationContainerReport
nodeRegistrationContainerReport =
- HddsTestUtils.createNodeRegistrationContainerReport(containers);
- queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
nodeRegistrationContainerReport);
- queue.fireEvent(SCMEvents.CONTAINER_REGISTRATION_REPORT,
nodeRegistrationContainerReport);
+ SCMDatanodeProtocolServer.NodeRegistrationContainerReport
nodeRegistrationContainerReport =
+ HddsTestUtils.createNodeRegistrationContainerReport(containers);
+ queue.fireEvent(SCMEvents.NODE_REGISTRATION_CONT_REPORT,
nodeRegistrationContainerReport);
+ queue.fireEvent(SCMEvents.CONTAINER_REGISTRATION_REPORT,
nodeRegistrationContainerReport);
+
- assertTrue(scmSafeModeManager.getInSafeMode());
+ assertTrue(scmSafeModeManager.getInSafeMode());
- firePipelineEvent(pipelineManager, pipeline);
+ firePipelineEvent(pipelineManager, pipeline);
- GenericTestUtils.waitFor(() -> !scmSafeModeManager.getInSafeMode(),
- 100, 1000 * 10);
- pipelineManager.close();
- } finally {
- config.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
- false);
- }
+ GenericTestUtils.waitFor(() -> !scmSafeModeManager.getInSafeMode(),
+ 100, 1000 * 10);
+ pipelineManager.close();
}
@Test
public void testPipelinesNotCreatedUntilPreCheckPasses() throws Exception {
int numOfDns = 5;
// enable pipeline check
- config.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK, true);
config.setInt(HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE, numOfDns);
config.setBoolean(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION,
true);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java
index 93d9d26de7..bf330e09e0 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestSCMPipelineBytesWrittenMetrics.java
@@ -30,7 +30,6 @@
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang3.RandomStringUtils;
-import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -63,8 +62,6 @@ public class TestSCMPipelineBytesWrittenMetrics {
@BeforeEach
public void setup() throws Exception {
conf = new OzoneConfiguration();
- conf.set(HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
- Boolean.TRUE.toString());
conf.setBoolean(OZONE_SCM_PIPELINE_AUTO_CREATE_FACTOR_ONE, false);
conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 1);
conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 10, TimeUnit.SECONDS);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
index e5ffe10fd2..cc7419648a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/safemode/TestSCMSafeModeWithPipelineRules.java
@@ -61,9 +61,6 @@ public void setup(int numDatanodes) throws Exception {
conf = new OzoneConfiguration();
conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL,
100, TimeUnit.MILLISECONDS);
- conf.setBoolean(
- HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_AVAILABILITY_CHECK,
- true);
conf.set(HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "10s");
conf.set(ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL, "10s");
conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
