somandal commented on code in PR #15266:
URL: https://github.com/apache/pinot/pull/15266#discussion_r2029695644
##########
pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/rebalance/TableRebalanceProgressStats.java:
##########
@@ -118,6 +126,209 @@ public RebalanceStateStats
getCurrentToTargetConvergence() {
return _currentToTargetConvergence;
}
+ public RebalanceProgressStats getRebalanceProgressStatsOverall() {
+ return _rebalanceProgressStatsOverall;
+ }
+
+ public RebalanceProgressStats getRebalanceProgressStatsCurrentStep() {
+ return _rebalanceProgressStatsCurrentStep;
+ }
+
+ /**
+ * Updates the overall and step progress stats based on the latest
calculated step's progress stats. This should
+ * be called during the EV-IS convergence trigger to ensure the overall
stats reflect the changes as they are made.
+ * @param currentStepStats latest step level stats calculated in this
iteration
+ */
+ public void updateOverallAndStepStatsFromLatestStepStats(
+ TableRebalanceProgressStats.RebalanceProgressStats currentStepStats) {
+ // Fetch the step level and overall stats that were calculated in the last
convergence check. These will be used
+ // to calculate the overall stats in the current convergence check
+ TableRebalanceProgressStats.RebalanceProgressStats previousStepStats =
getRebalanceProgressStatsCurrentStep();
+ TableRebalanceProgressStats.RebalanceProgressStats previousOverallStats =
getRebalanceProgressStatsOverall();
+
+ // Calculate the new segments added / deleted since the last step to track
overall change in segments. These are
+ // only new segments tracked by table rebalance (segments that aren't
tracked by rebalance convergence check are
+ // ignored because the convergence check does not wait for them to
complete)
+ int numAdditionalSegmentsAdded =
+ currentStepStats._totalSegmentsToBeAdded -
previousStepStats._totalSegmentsToBeAdded;
+ int numAdditionalSegmentsDeleted =
+ currentStepStats._totalSegmentsToBeDeleted -
previousStepStats._totalSegmentsToBeDeleted;
+
+ // Calculate the carry-over segment adds / deletes from the previous
rebalance step if any (can happen if
+ // bestEfforts=true or if there are deletions left over from the last
step) and the difference in the number of
+ // segments processed since the previous stats update.
+ // Example of what carry-over will look like:
+ // previousStepStats: totalSegmentsToBeAdded = 10
+ // previousStepStats: totalRemainingSegmentsToBeAdded = 15 (this can
only be larger than totalSegmentsToBeAdded
+ // if
carry-over segments exist)
+ // Carry-over: 15 - 10 = 5 segments -> which were added in the last IS
update by rebalance, but which didn't
+ // finish getting added before making the next assignment
IS update
+ // Using the above, we calculate the 'upperBoundOnSegmentsAdded/Deleted'
as the maximum segments to use when
+ // calculating the number of segments processed (number of segments that
got added / deleted in EV based on expected
+ // IS) since the last step. We want to skip including the carry-over
segments if any as they are tracked separately.
+ int upperBoundOnSegmentsAdded =
+ previousStepStats._totalRemainingSegmentsToBeAdded >
previousStepStats._totalSegmentsToBeAdded
+ ? previousStepStats._totalSegmentsToBeAdded :
previousStepStats._totalRemainingSegmentsToBeAdded;
Review Comment:
done, artifact from older code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org
For additional commands, e-mail: commits-h...@pinot.apache.org
