(doris) branch branch-2.0 updated: exclude today partition (#34507)

[dataroaring]

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-2.0 by this push:
     new ca7fb5161fa exclude today partition (#34507)
ca7fb5161fa is described below

commit ca7fb5161fac3b13d5073088fc5fac63e2824f52
Author: yujun <yu.jun.re...@gmail.com>
AuthorDate: Wed May 8 14:19:36 2024 +0800

    exclude today partition (#34507)
---
 .../doris/clone/DynamicPartitionScheduler.java     | 30 ++++++++++------------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
 
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 565f4c066f8..872be38de58 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -185,38 +185,28 @@ public class DynamicPartitionScheduler extends 
MasterDaemon {
         }
     }
 
-    private static int getBucketsNum(DynamicPartitionProperty property, 
OlapTable table) {
+    private static int getBucketsNum(DynamicPartitionProperty property, 
OlapTable table, String nowPartitionName) {
         if (!table.isAutoBucket()) {
             return property.getBuckets();
         }
 
         // auto bucket
         // get all history partitions
-        List<Partition> partitions = Lists.newArrayList();
+        ArrayList<Long> partitionSizeArray = Lists.newArrayList();
         RangePartitionInfo info = (RangePartitionInfo) 
(table.getPartitionInfo());
         List<Map.Entry<Long, PartitionItem>> idToItems = new 
ArrayList<>(info.getIdToItem(false).entrySet());
         idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem) 
o.getValue()).getItems().upperEndpoint()));
         for (Map.Entry<Long, PartitionItem> idToItem : idToItems) {
             Partition partition = table.getPartition(idToItem.getKey());
-            if (partition != null) {
-                partitions.add(partition);
-            }
-        }
-
-        // no exist history partition
-        if (partitions.size() == 0) {
-            return property.getBuckets();
-        }
-
-        ArrayList<Long> partitionSizeArray = Lists.newArrayList();
-        for (Partition partition : partitions) {
-            if (partition.getVisibleVersion() >= 2) {
+            // exclude current partition because its data isn't enough one 
week/day/hour.
+            if (partition != null && 
!partition.getName().equals(nowPartitionName)
+                    && partition.getVisibleVersion() >= 2) {
                 partitionSizeArray.add(partition.getAllDataSize(true));
             }
         }
 
         // no exist history partition data
-        if (partitionSizeArray.size() == 0) {
+        if (partitionSizeArray.isEmpty()) {
             return property.getBuckets();
         }
 
@@ -249,6 +239,12 @@ public class DynamicPartitionScheduler extends 
MasterDaemon {
         int hotPartitionNum = dynamicPartitionProperty.getHotPartitionNum();
         String storagePolicyName = dynamicPartitionProperty.getStoragePolicy();
 
+        String nowPartitionPrevBorder = 
DynamicPartitionUtil.getPartitionRangeString(
+                dynamicPartitionProperty, now, 0, partitionFormat);
+        String nowPartitionName = dynamicPartitionProperty.getPrefix()
+                + 
DynamicPartitionUtil.getFormattedPartitionName(dynamicPartitionProperty.getTimeZone(),
+                nowPartitionPrevBorder, 
dynamicPartitionProperty.getTimeUnit());
+
         for (; idx <= dynamicPartitionProperty.getEnd(); idx++) {
             String prevBorder = DynamicPartitionUtil.getPartitionRangeString(
                     dynamicPartitionProperty, now, idx, partitionFormat);
@@ -322,7 +318,7 @@ public class DynamicPartitionScheduler extends MasterDaemon 
{
 
             DistributionDesc distributionDesc = null;
             DistributionInfo distributionInfo = 
olapTable.getDefaultDistributionInfo();
-            int bucketsNum = getBucketsNum(dynamicPartitionProperty, 
olapTable);
+            int bucketsNum = getBucketsNum(dynamicPartitionProperty, 
olapTable, nowPartitionName);
             if (distributionInfo.getType() == 
DistributionInfo.DistributionInfoType.HASH) {
                 HashDistributionInfo hashDistributionInfo = 
(HashDistributionInfo) distributionInfo;
                 List<String> distColumnNames = new ArrayList<>();


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org