[doris] branch master updated: [enhancement](memory) reduce memory usage for failed broker loads (#16974)

[xuyang]

This is an automated email from the ASF dual-hosted git repository.

xuyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new d562428b1d [enhancement](memory) reduce memory usage for failed broker 
loads (#16974)
d562428b1d is described below

commit d562428b1d89c8aab433a06fa9d858ac6c50973b
Author: Zhengguo Yang <yangz...@gmail.com>
AuthorDate: Fri Feb 24 12:07:02 2023 +0800

    [enhancement](memory) reduce memory usage for failed broker loads (#16974)
    
    Reduce more memory usage for failed broker load msg in fe after pr  #15895
---
 .../src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java  | 5 ++++-
 fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java        | 5 ++++-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java 
b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
index d61e21e98d..9514eecaf3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadLoadingTask.java
@@ -45,6 +45,7 @@ import org.apache.logging.log4j.Logger;
 
 import java.util.List;
 import java.util.UUID;
+import java.util.stream.Collectors;
 
 public class LoadLoadingTask extends LoadTask {
     private static final Logger LOG = 
LogManager.getLogger(LoadLoadingTask.class);
@@ -177,7 +178,9 @@ public class LoadLoadingTask extends LoadTask {
                         curCoordinator.getLoadCounters(),
                         curCoordinator.getTrackingUrl(),
                         
TabletCommitInfo.fromThrift(curCoordinator.getCommitInfos()),
-                        
ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos()));
+                        
ErrorTabletInfo.fromThrift(curCoordinator.getErrorTabletInfos()
+                                
.stream().limit(Config.max_error_tablet_of_broker_load).collect(Collectors.toList())));
+                curCoordinator.getErrorTabletInfos().clear();
                 // Create profile of this task and add to the job profile.
                 createProfile(curCoordinator);
             } else {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java 
b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
index f26eec4478..929e06f3f7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java
@@ -1125,7 +1125,10 @@ public class Coordinator {
     private void updateErrorTabletInfos(List<TErrorTabletInfo> 
errorTabletInfos) {
         lock.lock();
         try {
-            this.errorTabletInfos.addAll(errorTabletInfos);
+            if (this.errorTabletInfos.size() <= 
Config.max_error_tablet_of_broker_load) {
+                
this.errorTabletInfos.addAll(errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load
+                        - 
this.errorTabletInfos.size()).collect(Collectors.toList()));
+            }
         } finally {
             lock.unlock();
         }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org