This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 219402d4c84 [opt](load) avoid over-flushing active memtables in memory
limiter (#52906)
219402d4c84 is described below
commit 219402d4c84fd02a06c724b8638c180877779f81
Author: Kaijie Chen <[email protected]>
AuthorDate: Fri Jul 11 10:06:15 2025 +0800
[opt](load) avoid over-flushing active memtables in memory limiter (#52906)
Related PR: #41018
Problem Summary:
Update the `_need_flush()` function to subtract both `_queue_mem_usage`
and `_flush_mem_usage` when deciding how much memory needs to be
flushed.
Previously, we only subtracted `_queue_mem_usage`, which could lead to
flushing more active memtables than necessary.
This change ensures that only the required amount of active memtable
memory is flushed, avoiding premature flushes.
This helps prevent creating small segments, which can hurt performance
and storage efficiency.
---
be/src/olap/memtable_memory_limiter.cpp | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/be/src/olap/memtable_memory_limiter.cpp
b/be/src/olap/memtable_memory_limiter.cpp
index f93b1ae16ed..d8efd4e26e1 100644
--- a/be/src/olap/memtable_memory_limiter.cpp
+++ b/be/src/olap/memtable_memory_limiter.cpp
@@ -40,6 +40,8 @@ bvar::Status<int64_t>
g_memtable_flush_memory("mm_limiter_mem_flush", 0);
bvar::Status<int64_t> g_memtable_load_memory("mm_limiter_mem_load", 0);
bvar::Status<int64_t> g_load_hard_mem_limit("mm_limiter_limit_hard", 0);
bvar::Status<int64_t> g_load_soft_mem_limit("mm_limiter_limit_soft", 0);
+bvar::Adder<int>
g_memtable_memory_limit_flush_memtable_count("mm_limiter_flush_memtable_count");
+bvar::LatencyRecorder
g_memtable_memory_limit_flush_size_bytes("mm_limiter_flush_size_bytes");
// Calculate the total memory limit of all load tasks on this BE
static int64_t calc_process_max_load_memory(int64_t process_mem_limit) {
@@ -114,7 +116,7 @@ int64_t MemTableMemoryLimiter::_need_flush() {
int64_t limit2 = _sys_avail_mem_less_than_warning_water_mark();
int64_t limit3 = _process_used_mem_more_than_soft_mem_limit();
int64_t need_flush = std::max(limit1, std::max(limit2, limit3));
- return need_flush - _queue_mem_usage;
+ return need_flush - _queue_mem_usage - _flush_mem_usage;
}
void
MemTableMemoryLimiter::handle_workload_group_memtable_flush(WorkloadGroupPtr
wg) {
@@ -178,6 +180,7 @@ void
MemTableMemoryLimiter::_handle_memtable_flush(WorkloadGroupPtr wg) {
<< ", active: " <<
PrettyPrinter::print_bytes(_active_mem_usage)
<< ", queue: " <<
PrettyPrinter::print_bytes(_queue_mem_usage)
<< ", flush: " <<
PrettyPrinter::print_bytes(_flush_mem_usage)
+ << ", need flush: " <<
PrettyPrinter::print_bytes(need_flush)
<< ", wg: " << (wg ? wg->debug_string() : "null");
if (VLOG_DEBUG_IS_ON) {
auto log_str = doris::ProcessProfile::instance()
@@ -281,6 +284,8 @@ int64_t
MemTableMemoryLimiter::_flush_active_memtables(uint64_t wg_id, int64_t n
}
mem_flushed += mem;
num_flushed += (mem > 0);
+ g_memtable_memory_limit_flush_memtable_count << 1;
+ g_memtable_memory_limit_flush_size_bytes << mem;
}
LOG(INFO) << "flushed " << num_flushed << " out of " <<
_active_writers.size()
<< " active writers, flushed size: " <<
PrettyPrinter::print_bytes(mem_flushed);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
