This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit e1951dcd8b1025198193d58f0bdf6702a6ab9cb9
Author: yujun <yu.jun.re...@gmail.com>
AuthorDate: Fri Sep 8 18:13:22 2023 +0800
[fix](trash) fix clean trash not working (#23936)
When executing admin clean trash, if the backend daemon clean thread is
cleaning trash, then SQL command will return immediately. But for the backend
daemon thread, it doesn't clean all the trashes, it clean only the expired
trashes.
Also if there's lots of trashes, the daemon clean thread will busy handling
trashes for a long time.
---
be/src/olap/olap_server.cpp | 4 ++++
be/src/olap/storage_engine.cpp | 5 ++++-
be/src/olap/storage_engine.h | 4 ++++
3 files changed, 12 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/olap_server.cpp b/be/src/olap/olap_server.cpp
index 1ccfae61b2..4697bbc793 100644
--- a/be/src/olap/olap_server.cpp
+++ b/be/src/olap/olap_server.cpp
@@ -295,6 +295,10 @@ void StorageEngine::_garbage_sweeper_thread_callback() {
// start clean trash and update usage.
Status res = start_trash_sweep(&usage);
+ if (res.ok() && _need_clean_trash.exchange(false,
std::memory_order_relaxed)) {
+ res = start_trash_sweep(&usage, true);
+ }
+
if (!res.ok()) {
LOG(WARNING) << "one or more errors occur when sweep trash."
<< "see previous message for detail. err code=" <<
res;
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index a701a7c8ed..83f5705b6a 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -635,10 +635,13 @@ Status StorageEngine::start_trash_sweep(double* usage,
bool ignore_guard) {
std::unique_lock<std::mutex> l(_trash_sweep_lock, std::defer_lock);
if (!l.try_lock()) {
LOG(INFO) << "trash and snapshot sweep is running.";
+ if (ignore_guard) {
+ _need_clean_trash.store(true, std::memory_order_relaxed);
+ }
return res;
}
- LOG(INFO) << "start trash and snapshot sweep.";
+ LOG(INFO) << "start trash and snapshot sweep. is_clean=" << ignore_guard;
const int32_t snapshot_expire = config::snapshot_expire_time_sec;
const int32_t trash_expire = config::trash_file_expire_time_sec;
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 86b132353d..0de5ff2668 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -487,6 +487,10 @@ private:
scoped_refptr<Thread> _async_publish_thread;
std::mutex _async_publish_mutex;
+ bool _clear_segment_cache = false;
+
+ std::atomic<bool> _need_clean_trash {false};
+
DISALLOW_COPY_AND_ASSIGN(StorageEngine);
};
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
