deardeng commented on code in PR #52801:
URL: https://github.com/apache/doris/pull/52801#discussion_r2189109331
##########
fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java:
##########
@@ -191,65 +192,99 @@ private static long getNextPartitionSize(ArrayList<Long>
historyPartitionsSize)
}
}
- private static int getBucketsNum(DynamicPartitionProperty property,
OlapTable table,
+ private static Pair<Integer, Integer>
getBucketsNum(DynamicPartitionProperty property, OlapTable table,
String partitionName, String nowPartitionName, boolean
executeFirstTime) {
// if execute first time, all partitions no contain data
if (!table.isAutoBucket() || executeFirstTime) {
- return property.getBuckets();
+ return Pair.of(property.getBuckets(), 0);
}
- // auto bucket
- // get all history partitions
+ List<Partition> partitions = getHistoricalPartitions(table,
nowPartitionName);
+ List<Long> visibleVersions = getVisibleVersions(partitions, table,
partitionName, property.getBuckets());
+
+ List<Partition> hasDataPartitions = filterDataPartitions(partitions,
visibleVersions);
+ if (hasDataPartitions.isEmpty()) {
+ return handleNoDataPartitions(table, partitionName,
property.getBuckets());
+ }
+
+ return calculateBuckets(hasDataPartitions);
+ }
+
+ private static List<Partition> getHistoricalPartitions(OlapTable table,
String nowPartitionName) {
RangePartitionInfo info = (RangePartitionInfo)
(table.getPartitionInfo());
List<Map.Entry<Long, PartitionItem>> idToItems = new
ArrayList<>(info.getIdToItem(false).entrySet());
idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem)
o.getValue()).getItems().upperEndpoint()));
- List<Partition> partitions = idToItems.stream()
+ return idToItems.stream()
.map(entry -> table.getPartition(entry.getKey()))
.filter(partition -> partition != null &&
!partition.getName().equals(nowPartitionName))
.collect(Collectors.toList());
- List<Long> visibleVersions = null;
+ }
+
+ private static List<Long> getVisibleVersions(List<Partition> partitions,
OlapTable table,
+ String partitionName, int
defaultBuckets) {
try {
- visibleVersions = Partition.getVisibleVersions(partitions);
+ return Partition.getVisibleVersions(partitions);
} catch (RpcException e) {
- LOG.warn("autobucket use property's buckets get visible version
fail, table: [{}-{}], "
+ LOG.warn("auto bucket use property's buckets get visible version
fail, table: [{}-{}], "
+ "partition: {}, buckets num: {}, exception: ",
- table.getName(), table.getId(), partitionName,
property.getBuckets(), e);
- return property.getBuckets();
+ table.getName(), table.getId(), partitionName,
defaultBuckets, e);
+ return Collections.emptyList(); // Return empty list to indicate
failure
}
+ }
- List<Partition> hasDataPartitions = Lists.newArrayList();
+ private static List<Partition> filterDataPartitions(List<Partition>
partitions, List<Long> visibleVersions) {
+ List<Partition> hasDataPartitions = new ArrayList<>();
for (int i = 0; i < partitions.size(); i++) {
if (visibleVersions.get(i) >= 2) {
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
