This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 3bb042e45c [fix](memtracker) Process physical mem check does not
include tc/jemalloc allocator cache (#12688)
3bb042e45c is described below
commit 3bb042e45cdfe81a025a779ac121b3ccf3934fc4
Author: Xinyi Zou <zouxiny...@gmail.com>
AuthorDate: Sat Sep 17 11:31:01 2022 +0800
[fix](memtracker) Process physical mem check does not include tc/jemalloc
allocator cache (#12688)
tcmalloc/jemalloc allocator cache does not participate in the mem check as
part of the process physical memory.
because new/malloc will trigger mem hook when using tcmalloc/jemalloc
allocator cache, but it may not actually alloc physical memory, which is not
expected in mem hook fail.
in addition:
The value of tcmalloc/jemalloc allocator cache is used as a mem tracker,
the parent is the process mem tracker, which is updated every 1s.
Modify the process default mem_limit to 90%. expect mem tracker to
effectively limit the memory usage of the process.
---
be/src/common/config.h | 2 +-
be/src/http/default_path_handlers.cpp | 7 +++---
be/src/runtime/exec_env.h | 5 +++++
be/src/runtime/exec_env_init.cpp | 1 +
be/src/runtime/memory/mem_tracker_limiter.h | 15 ++++++++++---
be/src/service/doris_main.cpp | 8 ++++++-
be/src/util/mem_info.cpp | 12 ++++++++--
be/src/util/mem_info.h | 28 +++++++++++++++++++-----
be/src/util/system_metrics.cpp | 3 ++-
docs/en/docs/admin-manual/config/be-config.md | 2 +-
docs/zh-CN/docs/admin-manual/config/be-config.md | 2 +-
11 files changed, 66 insertions(+), 19 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 3b8edf8daf..d426c85d7f 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -72,7 +72,7 @@ CONF_Int64(tc_max_total_thread_cache_bytes, "1073741824");
// defaults to bytes if no unit is given"
// must larger than 0. and if larger than physical memory size,
// it will be set to physical memory size.
-CONF_String(mem_limit, "80%");
+CONF_String(mem_limit, "90%");
// the port heartbeat service used
CONF_Int32(heartbeat_service_port, "9050");
diff --git a/be/src/http/default_path_handlers.cpp
b/be/src/http/default_path_handlers.cpp
index 2ce2b5faeb..ad55a97c47 100644
--- a/be/src/http/default_path_handlers.cpp
+++ b/be/src/http/default_path_handlers.cpp
@@ -31,6 +31,7 @@
#include "http/web_page_handler.h"
#include "runtime/memory/mem_tracker_limiter.h"
#include "util/debug_util.h"
+#include "util/perf_counters.h"
#include "util/pretty_printer.h"
#include "util/thread.h"
@@ -87,10 +88,8 @@ void mem_usage_handler(const WebPageHandler::ArgumentMap&
args, std::stringstrea
<<
PrettyPrinter::print(ExecEnv::GetInstance()->process_mem_tracker()->limit(),
TUnit::BYTES)
<< std::endl
- << "Mem Consumption: "
- <<
PrettyPrinter::print(ExecEnv::GetInstance()->process_mem_tracker()->consumption(),
- TUnit::BYTES)
- << std::endl
+ << "Physical Mem From Perf: "
+ << PrettyPrinter::print(PerfCounters::get_vm_rss(),
TUnit::BYTES) << std::endl
<< "</pre>";
(*output) << "<pre>";
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index 8e10db7d52..7be1c7e8d6 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -47,6 +47,7 @@ class ResultCache;
class LoadPathMgr;
class LoadStreamMgr;
class MemTrackerLimiter;
+class MemTracker;
class StorageEngine;
class MemTrackerTaskPool;
class PriorityThreadPool;
@@ -122,6 +123,9 @@ public:
_process_mem_tracker = tracker;
_process_mem_tracker_raw = tracker.get();
}
+ std::shared_ptr<MemTracker> allocator_cache_mem_tracker() {
+ return _allocator_cache_mem_tracker;
+ }
std::shared_ptr<MemTrackerLimiter> query_pool_mem_tracker() { return
_query_pool_mem_tracker; }
std::shared_ptr<MemTrackerLimiter> load_pool_mem_tracker() { return
_load_pool_mem_tracker; }
MemTrackerTaskPool* task_pool_mem_tracker_registry() { return
_task_pool_mem_tracker_registry; }
@@ -211,6 +215,7 @@ private:
// The ancestor for all trackers. Every tracker is visible from the
process down.
// Not limit total memory by process tracker, and it's just used to track
virtual memory of process.
std::shared_ptr<MemTrackerLimiter> _process_mem_tracker;
+ std::shared_ptr<MemTracker> _allocator_cache_mem_tracker;
MemTrackerLimiter* _process_mem_tracker_raw;
// The ancestor for all querys tracker.
std::shared_ptr<MemTrackerLimiter> _query_pool_mem_tracker;
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 907b7c26f1..5404394964 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -216,6 +216,7 @@ Status ExecEnv::_init_mem_tracker() {
}
#endif
+ _allocator_cache_mem_tracker =
std::make_shared<MemTracker>("Tc/JemallocAllocatorCache");
_query_pool_mem_tracker =
std::make_shared<MemTrackerLimiter>(-1, "QueryPool",
_process_mem_tracker);
REGISTER_HOOK_METRIC(query_mem_consumption,
diff --git a/be/src/runtime/memory/mem_tracker_limiter.h
b/be/src/runtime/memory/mem_tracker_limiter.h
index 5b23d4e0d6..f4ed944028 100644
--- a/be/src/runtime/memory/mem_tracker_limiter.h
+++ b/be/src/runtime/memory/mem_tracker_limiter.h
@@ -69,10 +69,19 @@ public:
// This is independent of the consumption value of the mem tracker,
which counts the virtual memory
// of the process malloc.
// for fast, expect MemInfo::initialized() to be true.
- if (PerfCounters::get_vm_rss() + bytes >= MemInfo::mem_limit()) {
+ // tcmalloc/jemalloc allocator cache does not participate in the mem
check as part of the process physical memory.
+ // because `new/malloc` will trigger mem hook when using
tcmalloc/jemalloc allocator cache,
+ // but it may not actually alloc physical memory, which is not
expected in mem hook fail.
+ //
+ // TODO: In order to ensure no OOM, currently reserve 200M, and then
use the free mem in /proc/meminfo to ensure no OOM.
+ if (PerfCounters::get_vm_rss() - MemInfo::allocator_cache_mem() +
bytes >=
+ MemInfo::mem_limit() ||
+ PerfCounters::get_vm_rss() + bytes >= MemInfo::hard_mem_limit()) {
auto st = Status::MemoryLimitExceeded(
- "process memory used {}, exceed limit {}, failed alloc
size {}",
- print_bytes(PerfCounters::get_vm_rss()),
print_bytes(MemInfo::mem_limit()),
+ "process memory used {}, tc/jemalloc cache {}, exceed
limit {}, failed alloc "
+ "size {}",
+ print_bytes(PerfCounters::get_vm_rss()),
+ print_bytes(MemInfo::allocator_cache_mem()),
print_bytes(MemInfo::mem_limit()),
print_bytes(bytes));
ExecEnv::GetInstance()->process_mem_tracker_raw()->print_log_usage(st.get_error_msg());
return st;
diff --git a/be/src/service/doris_main.cpp b/be/src/service/doris_main.cpp
index 3ef0b5b07a..0faeae639c 100644
--- a/be/src/service/doris_main.cpp
+++ b/be/src/service/doris_main.cpp
@@ -496,9 +496,15 @@ int main(int argc, char** argv) {
#if !defined(ADDRESS_SANITIZER) && !defined(LEAK_SANITIZER) &&
!defined(THREAD_SANITIZER) && \
!defined(USE_JEMALLOC)
- doris::MemInfo::refresh_current_mem();
+ doris::MemInfo::refresh_allocator_mem();
#endif
doris::PerfCounters::refresh_proc_status();
+ int64_t allocator_cache_mem_diff =
+ doris::MemInfo::allocator_cache_mem() -
+
doris::ExecEnv::GetInstance()->allocator_cache_mem_tracker()->consumption();
+ doris::ExecEnv::GetInstance()->allocator_cache_mem_tracker()->consume(
+ allocator_cache_mem_diff);
+ CONSUME_THREAD_MEM_TRACKER(allocator_cache_mem_diff);
// 1s clear the expired task mem tracker, a query mem tracker is about
57 bytes.
// this will cause coredump for ASAN build when running regression
test,
diff --git a/be/src/util/mem_info.cpp b/be/src/util/mem_info.cpp
index cf2aa58b4d..cf1cf0f8af 100644
--- a/be/src/util/mem_info.cpp
+++ b/be/src/util/mem_info.cpp
@@ -40,7 +40,13 @@ namespace doris {
bool MemInfo::_s_initialized = false;
int64_t MemInfo::_s_physical_mem = -1;
int64_t MemInfo::_s_mem_limit = -1;
-size_t MemInfo::_s_current_mem = 0;
+int64_t MemInfo::_s_hard_mem_limit = -1;
+size_t MemInfo::_s_allocator_physical_mem = 0;
+size_t MemInfo::_s_tcmalloc_pageheap_free_bytes = 0;
+size_t MemInfo::_s_tcmalloc_central_bytes = 0;
+size_t MemInfo::_s_tcmalloc_transfer_bytes = 0;
+size_t MemInfo::_s_tcmalloc_thread_bytes = 0;
+size_t MemInfo::_s_allocator_cache_mem = 0;
void MemInfo::init() {
// Read from /proc/meminfo
@@ -88,6 +94,7 @@ void MemInfo::init() {
bool is_percent = true;
_s_mem_limit = ParseUtil::parse_mem_spec(config::mem_limit, -1,
_s_physical_mem, &is_percent);
+ _s_hard_mem_limit = _s_physical_mem - std::min(209715200.0,
_s_physical_mem * 0.1); // 200M
LOG(INFO) << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem,
TUnit::BYTES);
_s_initialized = true;
@@ -100,7 +107,8 @@ std::string MemInfo::debug_string() {
stream << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem,
TUnit::BYTES)
<< std::endl;
stream << "Memory Limt: " << PrettyPrinter::print(_s_mem_limit,
TUnit::BYTES) << std::endl;
- stream << "Current Usage: " << PrettyPrinter::print(_s_current_mem,
TUnit::BYTES) << std::endl;
+ stream << "Current Usage: " <<
PrettyPrinter::print(_s_allocator_physical_mem, TUnit::BYTES)
+ << std::endl;
stream << "CGroup Info: " << util.debug_string() << std::endl;
return stream.str();
}
diff --git a/be/src/util/mem_info.h b/be/src/util/mem_info.h
index 7ce77301de..f29e328a8e 100644
--- a/be/src/util/mem_info.h
+++ b/be/src/util/mem_info.h
@@ -30,7 +30,7 @@ namespace doris {
// Provides the amount of physical memory available.
// Populated from /proc/meminfo.
-// TODO: Combine mem-info, cpu-info and disk-info into hardware-info?
+// TODO: Combine mem-info, cpu-info and disk-info into
hardware-info/perf_counters ?
class MemInfo {
public:
// Initialize MemInfo.
@@ -44,19 +44,31 @@ public:
return _s_physical_mem;
}
- static inline size_t current_mem() { return _s_current_mem; }
+ static inline size_t current_mem() { return _s_allocator_physical_mem; }
+ static inline size_t allocator_cache_mem() { return
_s_allocator_cache_mem; }
// Tcmalloc property `generic.total_physical_bytes` records the total
length of the virtual memory
// obtained by the process malloc, not the physical memory actually used
by the process in the OS.
- static inline void refresh_current_mem() {
+ static inline void refresh_allocator_mem() {
MallocExtension::instance()->GetNumericProperty("generic.total_physical_bytes",
- &_s_current_mem);
+
&_s_allocator_physical_mem);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.pageheap_free_bytes",
+
&_s_tcmalloc_pageheap_free_bytes);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.central_cache_free_bytes",
+
&_s_tcmalloc_central_bytes);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.transfer_cache_free_bytes",
+
&_s_tcmalloc_transfer_bytes);
+
MallocExtension::instance()->GetNumericProperty("tcmalloc.thread_cache_free_bytes",
+
&_s_tcmalloc_thread_bytes);
+ _s_allocator_cache_mem = _s_tcmalloc_pageheap_free_bytes +
_s_tcmalloc_central_bytes +
+ _s_tcmalloc_transfer_bytes +
_s_tcmalloc_thread_bytes;
}
static inline int64_t mem_limit() {
DCHECK(_s_initialized);
return _s_mem_limit;
}
+ static inline int64_t hard_mem_limit() { return _s_hard_mem_limit; }
static std::string debug_string();
@@ -64,7 +76,13 @@ private:
static bool _s_initialized;
static int64_t _s_physical_mem;
static int64_t _s_mem_limit;
- static size_t _s_current_mem;
+ static int64_t _s_hard_mem_limit;
+ static size_t _s_allocator_physical_mem;
+ static size_t _s_tcmalloc_pageheap_free_bytes;
+ static size_t _s_tcmalloc_central_bytes;
+ static size_t _s_tcmalloc_transfer_bytes;
+ static size_t _s_tcmalloc_thread_bytes;
+ static size_t _s_allocator_cache_mem;
};
} // namespace doris
diff --git a/be/src/util/system_metrics.cpp b/be/src/util/system_metrics.cpp
index d78c0588ad..9bc58687bc 100644
--- a/be/src/util/system_metrics.cpp
+++ b/be/src/util/system_metrics.cpp
@@ -25,6 +25,7 @@
#include "gutil/strtoint.h" // for atoi64
#include "util/doris_metrics.h"
#include "util/mem_info.h"
+#include "util/perf_counters.h"
namespace doris {
@@ -369,7 +370,7 @@ void SystemMetrics::_install_memory_metrics(MetricEntity*
entity) {
}
void SystemMetrics::_update_memory_metrics() {
- _memory_metrics->memory_allocated_bytes->set_value(MemInfo::current_mem());
+
_memory_metrics->memory_allocated_bytes->set_value(PerfCounters::get_vm_rss());
get_metrics_from_proc_vmstat();
}
diff --git a/docs/en/docs/admin-manual/config/be-config.md
b/docs/en/docs/admin-manual/config/be-config.md
index 8ffe3d58d3..1e9ea87bb8 100644
--- a/docs/en/docs/admin-manual/config/be-config.md
+++ b/docs/en/docs/admin-manual/config/be-config.md
@@ -838,7 +838,7 @@ The number of sliced tablets, plan the layout of the
tablet, and avoid too many
* Type: string
* Description: Limit the percentage of the server's maximum memory used by the
BE process. It is used to prevent BE memory from occupying to many the
machine's memory. This parameter must be greater than 0. When the percentage is
greater than 100%, the value will default to 100%.
-* Default value: 80%
+* Default value: 90%
### `memory_limitation_per_thread_for_schema_change`
diff --git a/docs/zh-CN/docs/admin-manual/config/be-config.md
b/docs/zh-CN/docs/admin-manual/config/be-config.md
index ccc034684f..a58bf80395 100644
--- a/docs/zh-CN/docs/admin-manual/config/be-config.md
+++ b/docs/zh-CN/docs/admin-manual/config/be-config.md
@@ -839,7 +839,7 @@ txn 管理器中每个 txn_partition_map 的最大 txns 数，这是一种自我
* 类型：string
* 描述：限制BE进程使用服务器最大内存百分比。用于防止BE内存挤占太多的机器内存，该参数必须大于0，当百分大于100%之后，该值会默认为100%。
-* 默认值：80%
+* 默认值：90%
### `memory_limitation_per_thread_for_schema_change`
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
