This is an automated email from the ASF dual-hosted git repository.
tingchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new d73aa1ee31 Add Segment Relocator Rebalance Configs to Controller Conf
(#13863)
d73aa1ee31 is described below
commit d73aa1ee31dfaa8a3319fffa29d040354fe87adf
Author: Prashant Pandey <84911643+suddend...@users.noreply.github.com>
AuthorDate: Mon Sep 9 23:06:52 2024 +0530
Add Segment Relocator Rebalance Configs to Controller Conf (#13863)
* Do best-efforts rebalance in SegmentRelocator
* Add rebalance configs to controller conf
* Rollback formatting changes
* Remove comment
* Addressed comments
* Checkstyle
* Rollback unrelated formatting changes
---
.../apache/pinot/controller/ControllerConf.java | 36 ++++++++++++++++++++++
.../helix/core/relocation/SegmentRelocator.java | 18 +++++++++++
.../pinot/controller/ControllerConfTest.java | 29 +++++++++++++++++
3 files changed, 83 insertions(+)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
index 1ae79f565a..f8a53f3db5 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
@@ -178,6 +178,17 @@ public class ControllerConf extends PinotConfiguration {
public static final String
DEPRECATED_SEGMENT_RELOCATOR_FREQUENCY_IN_SECONDS =
"controller.segment.relocator.frequencyInSeconds";
public static final String SEGMENT_RELOCATOR_FREQUENCY_PERIOD =
"controller.segment.relocator.frequencyPeriod";
+
+ //whether segment relocator should do a best-efforts rebalance. Default is
'true'
+ public static final String SEGMENT_RELOCATOR_BEST_EFFORTS =
"controller.segment.relocator.bestEfforts";
+ //For no-downtime rebalance, minimum number of replicas to keep alive
during rebalance, or maximum number of
+ // replicas allowed to be unavailable if value is negative. Default value
is -1
+ public static final String SEGMENT_RELOCATOR_MIN_AVAIL_REPLICAS =
+ "controller.segment.relocator.minAvailableReplicas";
+ public static final String SEGMENT_RELOCATOR_REASSIGN_INSTANCES =
"controller.segment.relocator.reassignInstances";
+ public static final String SEGMENT_RELOCATOR_BOOTSTRAP_SERVERS =
"controller.segment.relocator.bootstrap";
+ public static final String SEGMENT_RELOCATOR_DOWNTIME =
"controller.segment.relocator.downtime";
+
public static final String REBALANCE_CHECKER_FREQUENCY_PERIOD =
"controller.rebalance.checker.frequencyPeriod";
// Because segment level validation is expensive and requires heavy ZK
access, we run segment level validation
// with a separate interval
@@ -689,6 +700,31 @@ public class ControllerConf extends PinotConfiguration {
});
}
+ public boolean getSegmentRelocatorRebalanceConfigBestEfforts() {
+ return
Optional.ofNullable(getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_BEST_EFFORTS))
+ .map(Boolean::parseBoolean).orElse(true);
+ }
+
+ public int getSegmentRelocatorRebalanceConfigMinAvailReplicas() {
+ return
Optional.ofNullable(getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_MIN_AVAIL_REPLICAS))
+ .map(Integer::parseInt).orElse(-1);
+ }
+
+ public boolean getSegmentRelocatorRebalanceConfigReassignInstances() {
+ return
Optional.ofNullable(getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_REASSIGN_INSTANCES))
+ .map(Boolean::parseBoolean).orElse(false);
+ }
+
+ public boolean getSegmentRelocatorRebalanceConfigBootstrapServers() {
+ return
Optional.ofNullable(getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_BOOTSTRAP_SERVERS))
+ .map(Boolean::parseBoolean).orElse(false);
+ }
+
+ public boolean getSegmentRelocatorRebalanceConfigDowntime() {
+ return
Optional.ofNullable(getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_DOWNTIME))
+ .map(Boolean::parseBoolean).orElse(false);
+ }
+
public void setSegmentRelocatorFrequencyInSeconds(int
segmentRelocatorFrequencyInSeconds) {
setProperty(ControllerPeriodicTasksConf.DEPRECATED_SEGMENT_RELOCATOR_FREQUENCY_IN_SECONDS,
Integer.toString(segmentRelocatorFrequencyInSeconds));
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
index fc8567e178..f52bcf4447 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
@@ -66,6 +66,13 @@ public class SegmentRelocator extends
ControllerPeriodicTask<Void> {
private final boolean _enableLocalTierMigration;
private final int _serverAdminRequestTimeoutMs;
private final long _externalViewCheckIntervalInMs;
+
+ private final boolean _bestEffortsRebalance;
+ private final int _minAvailReplicasDuringRebalance;
+ private final boolean _reassignInstancesDuringRebalance;
+ private final boolean _bootstrapServersDuringRebalance;
+ private final boolean _downtime;
+
private final long _externalViewStabilizationTimeoutInMs;
private final Set<String> _waitingTables;
private final BlockingQueue<String> _waitingQueue;
@@ -86,6 +93,11 @@ public class SegmentRelocator extends
ControllerPeriodicTask<Void> {
Math.min(taskIntervalInMs,
config.getSegmentRelocatorExternalViewCheckIntervalInMs());
_externalViewStabilizationTimeoutInMs =
Math.min(taskIntervalInMs,
config.getSegmentRelocatorExternalViewStabilizationTimeoutInMs());
+ _bestEffortsRebalance =
config.getSegmentRelocatorRebalanceConfigBestEfforts();
+ _minAvailReplicasDuringRebalance =
config.getSegmentRelocatorRebalanceConfigMinAvailReplicas();
+ _reassignInstancesDuringRebalance =
config.getSegmentRelocatorRebalanceConfigReassignInstances();
+ _bootstrapServersDuringRebalance =
config.getSegmentRelocatorRebalanceConfigBootstrapServers();
+ _downtime = config.getSegmentRelocatorRebalanceConfigDowntime();
if (config.isSegmentRelocatorRebalanceTablesSequentially()) {
_waitingTables = ConcurrentHashMap.newKeySet();
_waitingQueue = new LinkedBlockingQueue<>();
@@ -169,6 +181,12 @@ public class SegmentRelocator extends
ControllerPeriodicTask<Void> {
rebalanceConfig.setExternalViewCheckIntervalInMs(_externalViewCheckIntervalInMs);
rebalanceConfig.setExternalViewStabilizationTimeoutInMs(_externalViewStabilizationTimeoutInMs);
rebalanceConfig.setUpdateTargetTier(TierConfigUtils.shouldRelocateToTiers(tableConfig));
+ //Do not fail the rebalance when the no-downtime contract cannot be
achieved
+ rebalanceConfig.setBestEfforts(_bestEffortsRebalance);
+ rebalanceConfig.setReassignInstances(_reassignInstancesDuringRebalance);
+ rebalanceConfig.setBootstrap(_bootstrapServersDuringRebalance);
+ rebalanceConfig.setMinAvailableReplicas(_minAvailReplicasDuringRebalance);
+ rebalanceConfig.setDowntime(_downtime);
try {
// Relocating segments to new tiers needs two sequential actions: table
rebalance and local tier migration.
diff --git
a/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerConfTest.java
b/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerConfTest.java
index 49494b914e..f566843760 100644
---
a/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerConfTest.java
+++
b/pinot-controller/src/test/java/org/apache/pinot/controller/ControllerConfTest.java
@@ -141,6 +141,35 @@ public class ControllerConfTest {
controllerConfig);
}
+ @Test
+ public void validateSegmentRelocatorRebalanceDefaultConfigs() {
+ //setup
+ Map<String, Object> controllerConfig = new HashMap<>();
+ ControllerConf conf = new ControllerConf(controllerConfig);
+
Assert.assertFalse(conf.getSegmentRelocatorRebalanceConfigBootstrapServers());
+
Assert.assertFalse(conf.getSegmentRelocatorRebalanceConfigReassignInstances());
+ Assert.assertTrue(conf.getSegmentRelocatorRebalanceConfigBestEfforts());
+ Assert.assertEquals(-1,
conf.getSegmentRelocatorRebalanceConfigMinAvailReplicas());
+ Assert.assertFalse(conf.getSegmentRelocatorRebalanceConfigDowntime());
+ }
+
+ @Test
+ public void validateSegmentRelocatorRebalanceConfigs() {
+ //setup
+ Map<String, Object> controllerConfig = new HashMap<>();
+ controllerConfig.put(SEGMENT_RELOCATOR_BEST_EFFORTS, true);
+ controllerConfig.put(SEGMENT_RELOCATOR_REASSIGN_INSTANCES, true);
+ controllerConfig.put(SEGMENT_RELOCATOR_MIN_AVAIL_REPLICAS, -2);
+ controllerConfig.put(SEGMENT_RELOCATOR_BOOTSTRAP_SERVERS, false);
+ controllerConfig.put(SEGMENT_RELOCATOR_DOWNTIME, true);
+ ControllerConf conf = new ControllerConf(controllerConfig);
+
Assert.assertFalse(conf.getSegmentRelocatorRebalanceConfigBootstrapServers());
+
Assert.assertTrue(conf.getSegmentRelocatorRebalanceConfigReassignInstances());
+ Assert.assertTrue(conf.getSegmentRelocatorRebalanceConfigBestEfforts());
+ Assert.assertEquals(-2,
conf.getSegmentRelocatorRebalanceConfigMinAvailReplicas());
+ Assert.assertTrue(conf.getSegmentRelocatorRebalanceConfigDowntime());
+ }
+
@Test
public void shouldBeAbleToDisableUsingNewConfig() {
Map<String, Object> controllerConfig = new HashMap<>();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
