This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 610f69432aa [improvement](segmentcache) limit segment cache by fd
limit or memory… (#39689)
610f69432aa is described below
commit 610f69432aa7af6ee31d4dac1d3021a76a2326cb
Author: Yongqiang YANG <98214048+dataroar...@users.noreply.github.com>
AuthorDate: Wed Aug 21 15:19:52 2024 +0800
[improvement](segmentcache) limit segment cache by fd limit or memory…
(#39689)
… (#39658)
remove a useless config.
---
be/src/common/config.cpp | 7 ++++---
be/src/common/config.h | 6 +++---
be/src/runtime/exec_env_init.cpp | 15 ++++++---------
3 files changed, 13 insertions(+), 15 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 461e328044d..656c5de3a98 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -40,6 +40,7 @@
#include "common/config.h"
#include "common/logging.h"
#include "common/status.h"
+#include "config.h"
#include "io/fs/file_writer.h"
#include "io/fs/local_file_system.h"
#include "util/cpu_info.h"
@@ -1068,10 +1069,10 @@ DEFINE_mInt32(schema_cache_capacity, "1024");
DEFINE_mInt32(schema_cache_sweep_time_sec, "100");
// max number of segment cache, default -1 for backward compatibility
fd_number*2/5
-DEFINE_mInt32(segment_cache_capacity, "-1");
-DEFINE_mInt32(estimated_num_columns_per_segment, "200");
+DEFINE_Int32(segment_cache_capacity, "-1");
+DEFINE_Int32(segment_cache_fd_percentage, "40");
DEFINE_mInt32(estimated_mem_per_column_reader, "1024");
-DEFINE_mInt32(segment_cache_memory_percentage, "2");
+DEFINE_Int32(segment_cache_memory_percentage, "2");
// enable feature binlog, default false
DEFINE_Bool(enable_feature_binlog, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 570fb61c5f3..d6080fd78b2 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1112,10 +1112,10 @@ DECLARE_mInt32(schema_cache_capacity);
DECLARE_mInt32(schema_cache_sweep_time_sec);
// max number of segment cache
-DECLARE_mInt32(segment_cache_capacity);
-DECLARE_mInt32(estimated_num_columns_per_segment);
-DECLARE_mInt32(estimated_mem_per_column_reader);
+DECLARE_Int32(segment_cache_capacity);
+DECLARE_Int32(segment_cache_fd_percentage);
DECLARE_Int32(segment_cache_memory_percentage);
+DECLARE_mInt32(estimated_mem_per_column_reader);
// enable binlog
DECLARE_Bool(enable_feature_binlog);
diff --git a/be/src/runtime/exec_env_init.cpp b/be/src/runtime/exec_env_init.cpp
index 7f4b0cbd874..fa2cef6e31d 100644
--- a/be/src/runtime/exec_env_init.cpp
+++ b/be/src/runtime/exec_env_init.cpp
@@ -451,21 +451,18 @@ Status ExecEnv::_init_mem_env() {
// SegmentLoader caches segments in rowset granularity. So the size of
// opened files will greater than segment_cache_capacity.
int64_t segment_cache_capacity = config::segment_cache_capacity;
- if (segment_cache_capacity < 0 || segment_cache_capacity > fd_number * 1 /
5) {
- segment_cache_capacity = fd_number * 1 / 5;
+ int64_t segment_cache_fd_limit = fd_number / 100 *
config::segment_cache_fd_percentage;
+ if (segment_cache_capacity < 0 || segment_cache_capacity >
segment_cache_fd_limit) {
+ segment_cache_capacity = segment_cache_fd_limit;
}
int64_t segment_cache_mem_limit =
MemInfo::mem_limit() / 100 *
config::segment_cache_memory_percentage;
- // config::segment_cache_memory_percentage;
- int64_t min_segment_cache_mem_limit =
- min(segment_cache_mem_limit, segment_cache_capacity *
-
config::estimated_num_columns_per_segment *
-
config::estimated_mem_per_column_reader);
- _segment_loader = new SegmentLoader(min_segment_cache_mem_limit,
segment_cache_capacity);
+
+ _segment_loader = new SegmentLoader(segment_cache_mem_limit,
segment_cache_capacity);
LOG(INFO) << "segment_cache_capacity <= fd_number * 1 / 5, fd_number: " <<
fd_number
<< " segment_cache_capacity: " << segment_cache_capacity
- << " min_segment_cache_mem_limit " <<
min_segment_cache_mem_limit;
+ << " min_segment_cache_mem_limit " << segment_cache_mem_limit;
_schema_cache = new SchemaCache(config::schema_cache_capacity);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
