[doris] branch branch-1.2-lts updated: [fix](cache file) fix auto run cache file cleaner when be start (#22724)

[yiguolei]

This is an automated email from the ASF dual-hosted git repository.

yiguolei pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
     new cf0a663b43 [fix](cache file) fix auto run cache file cleaner when be 
start (#22724)
cf0a663b43 is described below

commit cf0a663b4341e0866153ac734723e26ca1a2b0bc
Author: yujun <yu.jun.re...@gmail.com>
AuthorDate: Thu Aug 10 10:21:09 2023 +0800

    [fix](cache file) fix auto run cache file cleaner when be start (#22724)
    
    fix auto run cache file cleaner when be start
---
 be/src/common/config.h         |  2 ++
 be/src/olap/olap_server.cpp    | 16 +++++++++++++---
 be/src/olap/storage_engine.cpp |  8 ++++++++
 3 files changed, 23 insertions(+), 3 deletions(-)

diff --git a/be/src/common/config.h b/be/src/common/config.h
index 3aab83398f..34246da4cc 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -232,6 +232,8 @@ CONF_mInt32(tablet_rowset_stale_sweep_time_sec, "300");
 // garbage sweep policy
 CONF_Int32(max_garbage_sweep_interval, "3600");
 CONF_Int32(min_garbage_sweep_interval, "180");
+// garbage sweep every batch will sleep 1ms
+CONF_mInt32(garbage_sweep_batch_size, "100");
 CONF_mInt32(snapshot_expire_time_sec, "172800");
 // It is only a recommended value. When the disk space is insufficient,
 // the file storage period under trash dose not have to comply with this 
parameter.
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 28fd8e7629..848539d513 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -765,11 +765,21 @@ void StorageEngine::_cooldown_tasks_producer_callback() {
 }
 
 void StorageEngine::_cache_file_cleaner_tasks_producer_callback() {
-    int64_t interval = config::generate_cache_cleaner_task_interval_sec;
-    do {
+    while (true) {
+        int64_t interval = config::generate_cache_cleaner_task_interval_sec;
+        if (interval <= 0) {
+            interval = 10;
+        }
+        bool stop = 
_stop_background_threads_latch.wait_for(std::chrono::seconds(interval));
+        if (stop) {
+            break;
+        }
+        if (config::generate_cache_cleaner_task_interval_sec <= 0) {
+            continue;
+        }
         LOG(INFO) << "Begin to Clean cache files";
         FileCacheManager::instance()->gc_file_caches();
-    } while 
(!_stop_background_threads_latch.wait_for(std::chrono::seconds(interval)));
+    }
 }
 
 } // namespace doris
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 203ddafe0c..03182ecd5c 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -787,6 +787,7 @@ Status StorageEngine::_do_sweep(const std::string& 
scan_root, const time_t& loca
         return res;
     }
 
+    int curr_sweep_batch_size = 0;
     try {
         // Sort pathes by name, that is by delete time.
         std::vector<path> sorted_pathes;
@@ -822,6 +823,13 @@ Status StorageEngine::_do_sweep(const std::string& 
scan_root, const time_t& loca
                     res = Status::Error<OS_ERROR>();
                     continue;
                 }
+
+                curr_sweep_batch_size++;
+                if (config::garbage_sweep_batch_size > 0 &&
+                    curr_sweep_batch_size >= config::garbage_sweep_batch_size) 
{
+                    curr_sweep_batch_size = 0;
+                    std::this_thread::sleep_for(std::chrono::milliseconds(1));
+                }
             } else {
                 // Because files are ordered by filename, i.e. by create time, 
so all the left files are not expired.
                 break;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org