This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0d99b27d89 [fix] (mem tracker) Fix BE hangs at startup, stuck in
tcmalloc hook call ExecEnv::GetInstance() (#10517)
0d99b27d89 is described below
commit 0d99b27d893ed660a1618c0f6d5d3e75785385b4
Author: Kidd <107781942+k-i-...@users.noreply.github.com>
AuthorDate: Fri Jul 1 10:20:53 2022 +0800
[fix] (mem tracker) Fix BE hangs at startup, stuck in tcmalloc hook call
ExecEnv::GetInstance() (#10517)
1. Added flag exec_env_existed to indicate whether ExecEnv Instance is
created.
2. ThreadMemTrackerMgr::add_tracker fail when USE_MEM_TRACKER=OFF, add
USE_MEM_TRACKER compile option.
---
be/src/runtime/exec_env.h | 3 +++
be/src/runtime/tcmalloc_hook.h | 8 ++++----
be/src/runtime/thread_context.h | 8 ++------
be/src/runtime/thread_mem_tracker_mgr.h | 2 ++
4 files changed, 11 insertions(+), 10 deletions(-)
diff --git a/be/src/runtime/exec_env.h b/be/src/runtime/exec_env.h
index e0e09ae047..03e9971152 100644
--- a/be/src/runtime/exec_env.h
+++ b/be/src/runtime/exec_env.h
@@ -72,6 +72,8 @@ class ClientCache;
class HeartbeatFlags;
+static bool exec_env_existed = false;
+
// Execution environment for queries/plan fragments.
// Contains all required global structures, and handles to
// singleton services. Clients must call StartServices exactly
@@ -87,6 +89,7 @@ public:
/// we return the most recently created instance.
static ExecEnv* GetInstance() {
static ExecEnv s_exec_env;
+ exec_env_existed = true;
return &s_exec_env;
}
diff --git a/be/src/runtime/tcmalloc_hook.h b/be/src/runtime/tcmalloc_hook.h
index af60cdc178..dd0e1c788f 100644
--- a/be/src/runtime/tcmalloc_hook.h
+++ b/be/src/runtime/tcmalloc_hook.h
@@ -37,17 +37,17 @@
// destructor to control the behavior of consume can lead to unexpected
behavior,
// like this: if (LIKELY(doris::start_thread_mem_tracker)) {
void new_hook(const void* ptr, size_t size) {
- if (doris::tls_ctx()) {
+ if (doris::thread_local_ctx._init) {
doris::tls_ctx()->consume_mem(tc_nallocx(size, 0));
- } else if (doris::ExecEnv::GetInstance()->initialized()) {
+ } else if (doris::exec_env_existed &&
doris::ExecEnv::GetInstance()->initialized()) {
doris::MemTracker::get_process_tracker()->consume(tc_nallocx(size, 0));
}
}
void delete_hook(const void* ptr) {
- if (doris::tls_ctx()) {
+ if (doris::thread_local_ctx._init) {
doris::tls_ctx()->release_mem(tc_malloc_size(const_cast<void*>(ptr)));
- } else if (doris::ExecEnv::GetInstance()->initialized()) {
+ } else if (doris::exec_env_existed &&
doris::ExecEnv::GetInstance()->initialized()) {
doris::MemTracker::get_process_tracker()->release(tc_malloc_size(const_cast<void*>(ptr)));
}
}
diff --git a/be/src/runtime/thread_context.h b/be/src/runtime/thread_context.h
index 33478b19c5..bf347e0a28 100644
--- a/be/src/runtime/thread_context.h
+++ b/be/src/runtime/thread_context.h
@@ -214,6 +214,7 @@ public:
ThreadContext* get();
+ // TCMalloc hook is triggered during ThreadContext construction, which may
lead to deadlock.
bool _init = false;
private:
@@ -227,12 +228,7 @@ static ThreadContext* tls_ctx() {
if (tls != nullptr) {
return tls;
} else {
- if (thread_local_ctx._init) {
- return thread_local_ctx.get();
- } else {
- // TCMalloc hook is triggered during ThreadContext construction,
which may lead to deadlock.
- return nullptr;
- }
+ return thread_local_ctx.get();
}
}
diff --git a/be/src/runtime/thread_mem_tracker_mgr.h
b/be/src/runtime/thread_mem_tracker_mgr.h
index d8042ac0fa..ffc6fdc01f 100644
--- a/be/src/runtime/thread_mem_tracker_mgr.h
+++ b/be/src/runtime/thread_mem_tracker_mgr.h
@@ -292,11 +292,13 @@ inline void
ThreadMemTrackerMgr::noncache_try_consume(int64_t size) {
}
inline void ThreadMemTrackerMgr::add_tracker(const
std::shared_ptr<MemTracker>& mem_tracker) {
+#ifdef USE_MEM_TRACKER
DCHECK(_mem_trackers.find(mem_tracker->id()) == _mem_trackers.end()) <<
print_debug_string();
_mem_trackers[mem_tracker->id()] = mem_tracker;
DCHECK(_mem_trackers[mem_tracker->id()]) << print_debug_string();
_untracked_mems[mem_tracker->id()] = 0;
_mem_tracker_labels[mem_tracker->id()] = mem_tracker->label();
+#endif
}
inline std::shared_ptr<MemTracker> ThreadMemTrackerMgr::mem_tracker() {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
