[doris] branch master updated: [enhancement](stats) Limit analyze info count (#25576)

[morningman]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 6ce1087c231 [enhancement](stats) Limit analyze info count (#25576)
6ce1087c231 is described below

commit 6ce1087c231b490e34cccf33ceb45f3724c7e91e
Author: AKIRA <33112463+kikyou1...@users.noreply.github.com>
AuthorDate: Thu Oct 26 13:46:28 2023 +0800

    [enhancement](stats) Limit analyze info count (#25576)
    
    Each analyze job info and task info would not exceed 20000 after this PR. 
User could adjust this by FE conf param: analyze_record_limit
---
 .../src/main/java/org/apache/doris/common/Config.java   |  2 +-
 .../org/apache/doris/statistics/AnalysisManager.java    | 17 +++++++++++++----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java 
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index bbb064339f7..c50acaa0873 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2213,7 +2213,7 @@ public class Config extends ConfigBase {
             "控制统计信息的自动触发作业执行记录的持久化行数",
             "Determine the persist number of automatic triggered analyze job 
execution status"
     })
-    public static long auto_analyze_job_record_count = 20000;
+    public static long analyze_record_limit = 20000;
 
     @ConfField(description = {
             "Auto Buckets中最小的buckets数目",
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java 
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index ca17ca7b879..972e90c5831 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -86,6 +86,7 @@ import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
+import java.util.NavigableMap;
 import java.util.Optional;
 import java.util.Set;
 import java.util.StringJoiner;
@@ -112,10 +113,12 @@ public class AnalysisManager extends Daemon implements 
Writable {
     private AnalysisTaskExecutor taskExecutor;
 
     // Store task information in metadata.
-    private final Map<Long, AnalysisInfo> analysisTaskInfoMap = 
Collections.synchronizedMap(new TreeMap<>());
+    private final NavigableMap<Long, AnalysisInfo> analysisTaskInfoMap =
+            Collections.synchronizedNavigableMap(new TreeMap<>());
 
-    // Store job information in metadata
-    private final Map<Long, AnalysisInfo> analysisJobInfoMap = 
Collections.synchronizedMap(new TreeMap<>());
+    // Store job information in metadata.
+    private final NavigableMap<Long, AnalysisInfo> analysisJobInfoMap =
+            Collections.synchronizedNavigableMap(new TreeMap<>());
 
     // Tracking system submitted job, keep in mem only
     protected final Map<Long, AnalysisInfo> systemJobInfoMap = new 
ConcurrentHashMap<>();
@@ -819,10 +822,16 @@ public class AnalysisManager extends Daemon implements 
Writable {
     }
 
     public void replayCreateAnalysisJob(AnalysisInfo jobInfo) {
+        while (analysisJobInfoMap.size() >= Config.analyze_record_limit) {
+            
analysisJobInfoMap.remove(analysisJobInfoMap.pollFirstEntry().getKey());
+        }
         this.analysisJobInfoMap.put(jobInfo.jobId, jobInfo);
     }
 
     public void replayCreateAnalysisTask(AnalysisInfo taskInfo) {
+        while (analysisTaskInfoMap.size() >= Config.analyze_record_limit) {
+            
analysisTaskInfoMap.remove(analysisTaskInfoMap.pollFirstEntry().getKey());
+        }
         this.analysisTaskInfoMap.put(taskInfo.taskId, taskInfo);
     }
 
@@ -1075,7 +1084,7 @@ public class AnalysisManager extends Daemon implements 
Writable {
 
     protected SimpleQueue<AnalysisInfo> 
createSimpleQueue(Collection<AnalysisInfo> collection,
             AnalysisManager analysisManager) {
-        return new SimpleQueue<>(Config.auto_analyze_job_record_count,
+        return new SimpleQueue<>(Config.analyze_record_limit,
                 a -> {
                     // FE is not ready when replaying log and operations 
triggered by replaying
                     // shouldn't be logged again.


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org