This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3197f9b0655 [improve](load) reduce memory reserved in memtable limiter
(#37511)
3197f9b0655 is described below
commit 3197f9b0655f3af6f78786cb461b57eadfc42cbb
Author: Kaijie Chen <c...@apache.org>
AuthorDate: Wed Jul 10 22:20:48 2024 +0800
[improve](load) reduce memory reserved in memtable limiter (#37511)
## Proposed changes
#37174 increased max sys_mem_available_low_water_mark from 1.6G to 6.4G,
causing memtable memory limiter reserving too much memory.
This PR adds a BE config `memtable_limiter_reserved_memory_bytes`,
defaults to 800MB, which defines bytes reserved in memtable memory
limiter.
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 3 +++
be/src/olap/memtable_memory_limiter.cpp | 22 ++++++++++------------
be/src/olap/memtable_memory_limiter.h | 4 ++--
4 files changed, 17 insertions(+), 14 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index d016201ddfc..84f9d6f46bd 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -118,6 +118,8 @@ DEFINE_mInt32(double_resize_threshold, "23");
DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "6871947673");
+DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800");
+
// The size of the memory that gc wants to release each time, as a percentage
of the mem limit.
DEFINE_mString(process_minor_gc_size, "10%");
DEFINE_mString(process_full_gc_size, "20%");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index eec7d585f5d..2ed060cde6b 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -165,6 +165,9 @@ DECLARE_mInt32(double_resize_threshold);
// Turn down max. will use as much memory as possible.
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
+// reserve a small amount of memory so we do not trigger MinorGC
+DECLARE_Int64(memtable_limiter_reserved_memory_bytes);
+
// The size of the memory that gc wants to release each time, as a percentage
of the mem limit.
DECLARE_mString(process_minor_gc_size);
DECLARE_mString(process_full_gc_size);
diff --git a/be/src/olap/memtable_memory_limiter.cpp
b/be/src/olap/memtable_memory_limiter.cpp
index 1eaad31ec22..23b760284b8 100644
--- a/be/src/olap/memtable_memory_limiter.cpp
+++ b/be/src/olap/memtable_memory_limiter.cpp
@@ -77,20 +77,17 @@ void
MemTableMemoryLimiter::register_writer(std::weak_ptr<MemTableWriter> writer
_writers.push_back(writer);
}
-int64_t MemTableMemoryLimiter::_avail_mem_lack() {
+bool MemTableMemoryLimiter::_sys_avail_mem_less_than_warning_water_mark() {
// reserve a small amount of memory so we do not trigger MinorGC
- auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark();
- auto avail_mem_lack =
doris::MemInfo::sys_mem_available_warning_water_mark() -
- doris::GlobalMemoryArbitrator::sys_mem_available();
- return avail_mem_lack + reserved_mem;
+ return doris::GlobalMemoryArbitrator::sys_mem_available() <
+ doris::MemInfo::sys_mem_available_warning_water_mark() +
+ config::memtable_limiter_reserved_memory_bytes;
}
-int64_t MemTableMemoryLimiter::_proc_mem_extra() {
+bool MemTableMemoryLimiter::_process_used_mem_more_than_soft_mem_limit() {
// reserve a small amount of memory so we do not trigger MinorGC
- auto reserved_mem = doris::MemInfo::sys_mem_available_low_water_mark();
- auto proc_mem_extra =
- GlobalMemoryArbitrator::process_memory_usage() -
MemInfo::soft_mem_limit();
- return proc_mem_extra + reserved_mem;
+ return GlobalMemoryArbitrator::process_memory_usage() >
+ MemInfo::soft_mem_limit() -
config::memtable_limiter_reserved_memory_bytes;
}
bool MemTableMemoryLimiter::_soft_limit_reached() {
@@ -98,8 +95,9 @@ bool MemTableMemoryLimiter::_soft_limit_reached() {
}
bool MemTableMemoryLimiter::_hard_limit_reached() {
- return _mem_tracker->consumption() >= _load_hard_mem_limit ||
_avail_mem_lack() >= 0 ||
- _proc_mem_extra() >= 0;
+ return _mem_tracker->consumption() >= _load_hard_mem_limit ||
+ _sys_avail_mem_less_than_warning_water_mark() ||
+ _process_used_mem_more_than_soft_mem_limit();
}
bool MemTableMemoryLimiter::_load_usage_low() {
diff --git a/be/src/olap/memtable_memory_limiter.h
b/be/src/olap/memtable_memory_limiter.h
index 38cd83e61dc..2e8271bab35 100644
--- a/be/src/olap/memtable_memory_limiter.h
+++ b/be/src/olap/memtable_memory_limiter.h
@@ -51,8 +51,8 @@ public:
int64_t mem_usage() const { return _mem_usage; }
private:
- static int64_t _avail_mem_lack();
- static int64_t _proc_mem_extra();
+ static inline bool _sys_avail_mem_less_than_warning_water_mark();
+ static inline bool _process_used_mem_more_than_soft_mem_limit();
bool _soft_limit_reached();
bool _hard_limit_reached();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
