This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 50af594 [MemLimit] Normalize the setting of mem limit (#3033)
50af594 is described below
commit 50af594c66ef2482952b032079f7df2005c93317
Author: Mingyu Chen <morningman....@gmail.com>
AuthorDate: Thu Mar 5 08:47:45 2020 +0800
[MemLimit] Normalize the setting of mem limit (#3033)
Normalize the setting of mem limit to avoid some unexpected exception.
For example, use may not setting query mem limit in query plan, which
may cause BE crash.
---
be/src/common/config.h | 2 ++
be/src/runtime/exec_env_init.cpp | 18 ++++++++------
be/src/runtime/plan_fragment_executor.cpp | 40 +++++++++++++++++--------------
3 files changed, 35 insertions(+), 25 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 73b6f77..8f56a25 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -48,6 +48,8 @@ namespace config {
// ('<int>[bB]?'), megabytes ('<float>[mM]'), gigabytes ('<float>[gG]'),
// or percentage of the physical memory ('<int>%').
// defaults to bytes if no unit is given"
+ // must larger than 0. and if larger than physical memory size,
+ // it will be set to physical memory size.
CONF_String(mem_limit, "80%");
// the port heartbeat service used
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 47cc38b..dd241a0 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -132,7 +132,7 @@ Status ExecEnv::_init_mem_tracker() {
std::stringstream ss;
// --mem_limit="" means no memory limit
bytes_limit = ParseUtil::parse_mem_spec(config::mem_limit, &is_percent);
- if (bytes_limit < 0) {
+ if (bytes_limit <= 0) {
ss << "Failed to parse mem limit from '" + config::mem_limit + "'.";
return Status::InternalError(ss.str());
}
@@ -161,19 +161,23 @@ Status ExecEnv::_init_mem_tracker() {
_init_buffer_pool(config::min_buffer_size, buffer_pool_limit,
clean_pages_limit);
- // Limit of 0 means no memory limit.
- if (bytes_limit > 0) {
- _mem_tracker = new MemTracker(bytes_limit);
- }
-
if (bytes_limit > MemInfo::physical_mem()) {
LOG(WARNING) << "Memory limit "
<< PrettyPrinter::print(bytes_limit, TUnit::BYTES)
<< " exceeds physical memory of "
<< PrettyPrinter::print(MemInfo::physical_mem(),
- TUnit::BYTES);
+ TUnit::BYTES)
+ << ". Using physical memory instead";
+ bytes_limit = MemInfo::physical_mem();
+ }
+
+ if (bytes_limit <= 0) {
+ ss << "Invalid mem limit: " << bytes_limit;
+ return Status::InternalError(ss.str());
}
+ _mem_tracker = new MemTracker(bytes_limit);
+
LOG(INFO) << "Using global memory limit: " <<
PrettyPrinter::print(bytes_limit, TUnit::BYTES);
RETURN_IF_ERROR(_disk_io_mgr->init(_mem_tracker));
RETURN_IF_ERROR(_tmp_file_mgr->init(DorisMetrics::metrics()));
diff --git a/be/src/runtime/plan_fragment_executor.cpp
b/be/src/runtime/plan_fragment_executor.cpp
index a01ffe7..18cfb0d 100644
--- a/be/src/runtime/plan_fragment_executor.cpp
+++ b/be/src/runtime/plan_fragment_executor.cpp
@@ -113,24 +113,28 @@ Status PlanFragmentExecutor::prepare(const
TExecPlanFragmentParams& request) {
//
_runtime_state->mem_trackers()->push_back(_exec_env->process_mem_tracker());
// }
- if (request.query_options.mem_limit > 0) {
- // we have a per-query limit
- int64_t bytes_limit = request.query_options.mem_limit;
- // NOTE: this MemTracker only for olap
- _mem_tracker.reset(
- new MemTracker(bytes_limit, "fragment mem-limit",
_exec_env->process_mem_tracker()));
- _runtime_state->set_fragment_mem_tracker(_mem_tracker.get());
-
- if (bytes_limit > MemInfo::physical_mem()) {
- LOG(WARNING) << "Memory limit "
- << PrettyPrinter::print(bytes_limit, TUnit::BYTES)
- << " exceeds physical memory of "
- << PrettyPrinter::print(MemInfo::physical_mem(),
TUnit::BYTES);
- }
-
- LOG(INFO) << "Using query memory limit: "
- << PrettyPrinter::print(bytes_limit, TUnit::BYTES);
- }
+ int64_t bytes_limit = request.query_options.mem_limit;
+ if (bytes_limit <= 0) {
+ // sometimes the request does not set the query mem limit, we use
default one.
+ // TODO(cmy): we should not allow request without query mem limit.
+ bytes_limit = 2 * 1024 * 1024 * 1024L;
+ }
+
+ if (bytes_limit > _exec_env->process_mem_tracker()->limit()) {
+ LOG(WARNING) << "Query memory limit "
+ << PrettyPrinter::print(bytes_limit, TUnit::BYTES)
+ << " exceeds process memory limit of "
+ << PrettyPrinter::print(_exec_env->process_mem_tracker()->limit(),
TUnit::BYTES)
+ << ". Using process memory limit instead";
+ bytes_limit = _exec_env->process_mem_tracker()->limit();
+ }
+ // NOTE: this MemTracker only for olap
+ _mem_tracker.reset(
+ new MemTracker(bytes_limit, "fragment mem-limit",
_exec_env->process_mem_tracker()));
+ _runtime_state->set_fragment_mem_tracker(_mem_tracker.get());
+
+ LOG(INFO) << "Using query memory limit: "
+ << PrettyPrinter::print(bytes_limit, TUnit::BYTES);
RETURN_IF_ERROR(_runtime_state->create_block_mgr());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
