This is an automated email from the ASF dual-hosted git repository.
xbli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6c0215834b Expose more parameters in rebalance config via controller
config (#15509)
6c0215834b is described below
commit 6c0215834bf3e6203d1246bbb0b49e01b369b6d4
Author: Jhow <44998515+j-howhu...@users.noreply.github.com>
AuthorDate: Fri Apr 11 15:56:38 2025 -0700
Expose more parameters in rebalance config via controller config (#15509)
* Update rebalance config in SegmentRelocator
* Rename config
---
.../org/apache/pinot/controller/ControllerConf.java | 20 +++++++++++++++++++-
.../helix/core/relocation/SegmentRelocator.java | 6 ++++++
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
index 29ebb084f0..0892564ed2 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/ControllerConf.java
@@ -198,6 +198,11 @@ public class ControllerConf extends PinotConfiguration {
"controller.segmentRelocator.enableLocalTierMigration";
public static final String SEGMENT_RELOCATOR_REBALANCE_TABLES_SEQUENTIALLY
=
"controller.segmentRelocator.rebalanceTablesSequentially";
+ public static final String SEGMENT_RELOCATOR_REBALANCE_INCLUDE_CONSUMING =
+ "controller.segmentRelocator.includeConsuming";
+ // Available options are: "ENABLE", "DISABLE", "DEFAULT"
+ public static final String
SEGMENT_RELOCATOR_REBALANCE_MINIMIZE_DATA_MOVEMENT =
+ "controller.segmentRelocator.minimizeDataMovement";
public static final String REBALANCE_CHECKER_FREQUENCY_PERIOD =
"controller.rebalance.checker.frequencyPeriod";
// Because segment level validation is expensive and requires heavy ZK
access, we run segment level validation
@@ -800,6 +805,20 @@ public class ControllerConf extends PinotConfiguration {
return
getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_REBALANCE_TABLES_SEQUENTIALLY,
false);
}
+ public boolean isSegmentRelocatorIncludingConsuming() {
+ return
getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_REBALANCE_INCLUDE_CONSUMING,
false);
+ }
+
+ public RebalanceConfig.MinimizeDataMovementOptions
getSegmentRelocatorRebalanceMinimizeDataMovement() {
+ String value =
getProperty(ControllerPeriodicTasksConf.SEGMENT_RELOCATOR_REBALANCE_MINIMIZE_DATA_MOVEMENT,
+ RebalanceConfig.MinimizeDataMovementOptions.ENABLE.name());
+ try {
+ return
RebalanceConfig.MinimizeDataMovementOptions.valueOf(value.toUpperCase());
+ } catch (IllegalArgumentException e) {
+ return RebalanceConfig.MinimizeDataMovementOptions.ENABLE;
+ }
+ }
+
public boolean tieredSegmentAssignmentEnabled() {
return getProperty(CONTROLLER_ENABLE_TIERED_SEGMENT_ASSIGNMENT, false);
}
@@ -1109,7 +1128,6 @@ public class ControllerConf extends PinotConfiguration {
setProperty(ControllerPeriodicTasksConf.ENABLE_UNTRACKED_SEGMENT_DELETION,
untrackedSegmentDeletionEnabled);
}
-
public long getPinotTaskManagerInitialDelaySeconds() {
return getPeriodicTaskInitialDelayInSeconds();
}
diff --git
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
index 13b4bae7f1..eaed36b00a 100644
---
a/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
+++
b/pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/relocation/SegmentRelocator.java
@@ -74,6 +74,8 @@ public class SegmentRelocator extends
ControllerPeriodicTask<Void> {
private final boolean _bestEfforts;
private final long _externalViewCheckIntervalInMs;
private final long _externalViewStabilizationTimeoutInMs;
+ private final boolean _includeConsuming;
+ private final RebalanceConfig.MinimizeDataMovementOptions
_minimizeDataMovement;
private final Set<String> _waitingTables;
private final BlockingQueue<String> _waitingQueue;
@@ -94,6 +96,8 @@ public class SegmentRelocator extends
ControllerPeriodicTask<Void> {
_downtime = config.getSegmentRelocatorDowntime();
_minAvailableReplicas = config.getSegmentRelocatorMinAvailableReplicas();
_bestEfforts = config.getSegmentRelocatorBestEfforts();
+ _includeConsuming = config.isSegmentRelocatorIncludingConsuming();
+ _minimizeDataMovement =
config.getSegmentRelocatorRebalanceMinimizeDataMovement();
// Best effort to let inner part of the task run no longer than the task
interval, although not enforced strictly.
long taskIntervalInMs = config.getSegmentRelocatorFrequencyInSeconds() *
1000L;
_externalViewCheckIntervalInMs =
@@ -187,6 +191,8 @@ public class SegmentRelocator extends
ControllerPeriodicTask<Void> {
rebalanceConfig.setExternalViewCheckIntervalInMs(_externalViewCheckIntervalInMs);
rebalanceConfig.setExternalViewStabilizationTimeoutInMs(_externalViewStabilizationTimeoutInMs);
rebalanceConfig.setUpdateTargetTier(TierConfigUtils.shouldRelocateToTiers(tableConfig));
+ rebalanceConfig.setIncludeConsuming(_includeConsuming);
+ rebalanceConfig.setMinimizeDataMovement(_minimizeDataMovement);
try {
// Relocating segments to new tiers needs two sequential actions: table
rebalance and local tier migration.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
