somandal commented on code in PR #16017: URL: https://github.com/apache/pinot/pull/16017#discussion_r2136567027
########## pinot-controller/src/main/java/org/apache/pinot/controller/helix/core/assignment/segment/MultiTierStrictRealtimeSegmentAssignment.java: ########## @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.pinot.controller.helix.core.assignment.segment; + +import com.google.common.base.Preconditions; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import javax.annotation.Nullable; +import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.lang3.tuple.Pair; +import org.apache.pinot.common.assignment.InstancePartitions; +import org.apache.pinot.common.tier.Tier; +import org.apache.pinot.controller.helix.core.rebalance.RebalanceConfig; +import org.apache.pinot.spi.config.table.assignment.InstancePartitionsType; +import org.apache.pinot.spi.utils.CommonConstants; + + +/** + * This segment assignment policy allows the table to use multiple tiers when rebalancing table. This can be used for + * dedup table, whose segments out of TTL can be moved to new tiers without messing up the dedup metadata tracked on + * the CONSUMING servers. + */ +public class MultiTierStrictRealtimeSegmentAssignment extends BaseStrictRealtimeSegmentAssignment { + @Override + public Map<String, Map<String, String>> rebalanceTable(Map<String, Map<String, String>> currentAssignment, + Map<InstancePartitionsType, InstancePartitions> instancePartitionsMap, @Nullable List<Tier> sortedTiers, + @Nullable Map<String, InstancePartitions> tierInstancePartitionsMap, RebalanceConfig config) { + Preconditions.checkState(instancePartitionsMap.size() == 1, "One instance partition type should be provided"); + InstancePartitions instancePartitions = instancePartitionsMap.get(InstancePartitionsType.CONSUMING); + Preconditions.checkState(instancePartitions != null, "Failed to find CONSUMING instance partitions for table: %s", + _tableNameWithType); + Preconditions.checkArgument(config.isIncludeConsuming(), + "Consuming segment must be included when rebalancing table: %s using multi-tier " + + "StrictRealtimeSegmentAssignment", _tableNameWithType); + boolean bootstrap = config.isBootstrap(); + _logger.info("Rebalancing table: {} with instance partitions: {}, bootstrap: {}", _tableNameWithType, + instancePartitions, bootstrap); + // Rebalance tiers first. Only completed segments are moved to other tiers. + Pair<List<Map<String, Map<String, String>>>, Map<String, Map<String, String>>> pair = + rebalanceTiers(currentAssignment, sortedTiers, tierInstancePartitionsMap, bootstrap, Review Comment: just for my understanding - does this handle the case where only segments past the metadataTTL can be moved for dedup? or is that a non-concern? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@pinot.apache.org For additional commands, e-mail: commits-h...@pinot.apache.org
