(doris) branch branch-3.0 updated: branch-3.0: [fix](table) Breaking up dynamic partition table scheduling consuming peak CPU consumption #48577 (#48879)

[dataroaring]

This is an automated email from the ASF dual-hosted git repository.

dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-3.0 by this push:
     new 5124dc82f88 branch-3.0: [fix](table) Breaking up dynamic partition 
table scheduling consuming peak CPU consumption #48577 (#48879)
5124dc82f88 is described below

commit 5124dc82f88d423772a83437cb9a3fd148c95922
Author: github-actions[bot] 
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Mar 11 11:45:28 2025 +0800

    branch-3.0: [fix](table) Breaking up dynamic partition table scheduling 
consuming peak CPU consumption #48577 (#48879)
    
    Cherry-picked from #48577
    
    Co-authored-by: deardeng <deng...@selectdb.com>
---
 .../src/main/java/org/apache/doris/common/Config.java      |  8 ++++++++
 .../org/apache/doris/clone/DynamicPartitionScheduler.java  | 14 ++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java 
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 90a1951fc45..33062fb164c 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1324,6 +1324,14 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true, masterOnly = true)
     public static long dynamic_partition_check_interval_seconds = 600;
 
+    /**
+     * When scheduling dynamic partition tables,
+     * the execution interval of each table to prevent excessive consumption 
of FE CPU at the same time
+     * default is 0
+     */
+    @ConfField(mutable = true, masterOnly = true)
+    public static long dynamic_partition_step_interval_ms = 0;
+
     /**
      * If set to true, dynamic partition feature will open
      */
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
 
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index e875bcf47d8..8e5f46573ee 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -671,6 +671,20 @@ public class DynamicPartitionScheduler extends 
MasterDaemon {
                 }
                 cloudBatchAfterCreatePartitions(executeFirstTime, partsInfo,
                         addPartitionClauses, db, olapTable, indexIds, 
tableName);
+
+                // ATTN: Breaking up dynamic partition table scheduling, 
consuming peak CPU consumption
+                if (!executeFirstTime && !addPartitionClauses.isEmpty()
+                        && Config.dynamic_partition_step_interval_ms > 0) {
+                    long sleep = Config.dynamic_partition_step_interval_ms;
+                    if (sleep > 1800 * 1000) {
+                        LOG.warn("fe conf dynamic_partition_step_interval_ms 
bigger than 1800s, plz check it");
+                    }
+                    try {
+                        Thread.sleep(sleep);
+                    } catch (InterruptedException e) {
+                        LOG.warn("sleep err", e);
+                    }
+                }
             }
         }
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org