This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 6714936f8ba [pick](branch-2.1) pick #39962 #40304 (#44931)
6714936f8ba is described below
commit 6714936f8ba8335ba35b729d5cf2083272779629
Author: Xinyi Zou <zouxi...@selectdb.com>
AuthorDate: Wed Dec 4 17:56:58 2024 +0800
[pick](branch-2.1) pick #39962 #40304 (#44931)
---
be/src/common/config.cpp | 7 ++++++-
be/src/common/config.h | 8 ++++----
be/src/runtime/memory/mem_tracker.h | 5 +++--
be/src/runtime/memory/mem_tracker_limiter.cpp | 1 -
be/src/util/mem_info.cpp | 12 ++++++------
5 files changed, 19 insertions(+), 14 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 4931b61940d..fd0282586e6 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -142,7 +142,12 @@ DEFINE_mInt32(max_fill_rate, "2");
DEFINE_mInt32(double_resize_threshold, "23");
-DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "6871947673");
+// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit
byte, default -1.
+// if it is -1, then low water mark = min(MemTotal - MemLimit, MemTotal * 5%),
which is 3.2G on a 64G machine.
+// Turn up max. more memory buffers will be reserved for Memory GC.
+// Turn down max. will use as much memory as possible.
+// note that: `max_` prefix should be removed, but keep it for compatibility.
+DEFINE_Int64(max_sys_mem_available_low_water_mark_bytes, "-1");
DEFINE_Int64(memtable_limiter_reserved_memory_bytes, "838860800");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 15e384e7dfb..f50462f20c2 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -178,11 +178,11 @@ DECLARE_mInt32(max_fill_rate);
DECLARE_mInt32(double_resize_threshold);
-// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit
byte, default 6.4G,
-// actual low water mark=min(6.4G, MemTotal * 5%), avoid wasting too much
memory on machines
-// with large memory larger than 128G.
-// Turn up max. On machines with more than 128G memory, more memory buffers
will be reserved for Full GC.
+// The maximum low water mark of the system `/proc/meminfo/MemAvailable`, Unit
byte, default -1.
+// if it is -1, then low water mark = min(MemTotal - MemLimit, MemTotal * 5%),
which is 3.2G on a 64G machine.
+// Turn up max. more memory buffers will be reserved for Memory GC.
// Turn down max. will use as much memory as possible.
+// note that: `max_` prefix should be removed, but keep it for compatibility.
DECLARE_Int64(max_sys_mem_available_low_water_mark_bytes);
// reserve a small amount of memory so we do not trigger MinorGC
diff --git a/be/src/runtime/memory/mem_tracker.h
b/be/src/runtime/memory/mem_tracker.h
index 8a574398e0e..8a977e49388 100644
--- a/be/src/runtime/memory/mem_tracker.h
+++ b/be/src/runtime/memory/mem_tracker.h
@@ -146,8 +146,6 @@ public:
// Creates and adds the tracker to the mem_tracker_pool.
MemTracker(const std::string& label, MemTrackerLimiter* parent = nullptr);
- // For MemTrackerLimiter
- MemTracker() { _parent_group_num = -1; }
virtual ~MemTracker();
@@ -203,6 +201,9 @@ public:
}
protected:
+ // Only used by MemTrackerLimiter
+ MemTracker() { _parent_group_num = -1; }
+
void bind_parent(MemTrackerLimiter* parent);
Type _type;
diff --git a/be/src/runtime/memory/mem_tracker_limiter.cpp
b/be/src/runtime/memory/mem_tracker_limiter.cpp
index e6144aa05fe..63391203c5d 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.cpp
+++ b/be/src/runtime/memory/mem_tracker_limiter.cpp
@@ -336,7 +336,6 @@ void
MemTrackerLimiter::make_process_snapshots(std::vector<MemTracker::Snapshot>
snapshot.cur_consumption =
GlobalMemoryArbitrator::process_reserved_memory();
snapshot.peak_consumption = -1;
(*snapshots).emplace_back(snapshot);
- all_trackers_mem_sum += GlobalMemoryArbitrator::process_reserved_memory();
snapshot.type = "overview";
snapshot.label = "sum_of_all_trackers"; // is virtual memory
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index b820234e712..b1bcfdcc56b 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -317,12 +317,12 @@ void MemInfo::init() {
//
https://serverfault.com/questions/940196/why-is-memavailable-a-lot-less-than-memfreebufferscached
//
https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?id=34e431b0ae398fc54ea69ff85ec700722c9da773
//
- // upper sys_mem_available_low_water_mark, avoid wasting too much
memory.
- _s_sys_mem_available_low_water_mark = std::max<int64_t>(
- std::min<int64_t>(std::min<int64_t>(_s_physical_mem -
_s_mem_limit,
- int64_t(_s_physical_mem *
0.05)),
-
config::max_sys_mem_available_low_water_mark_bytes),
- 0);
+ // smaller sys_mem_available_low_water_mark can avoid wasting too much
memory.
+ _s_sys_mem_available_low_water_mark =
+ config::max_sys_mem_available_low_water_mark_bytes != -1
+ ? config::max_sys_mem_available_low_water_mark_bytes
+ : std::min<int64_t>(_s_physical_mem - _s_mem_limit,
+ int64_t(_s_physical_mem * 0.05));
_s_sys_mem_available_warning_water_mark =
_s_sys_mem_available_low_water_mark * 2;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
