This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 3165a7b2c731d56f4471056c1764c4365bfb31b5
Author: zhengyu <freeman.zhang1...@gmail.com>
AuthorDate: Fri Jun 14 09:25:58 2024 +0800
[feature](cloud) support file cache only cache index pages (#36273)
give user an option to only cache index pages instead of all data pages
Signed-off-by: freemandealer <freeman.zhang1...@gmail.com>
---
be/src/common/config.cpp | 2 ++
be/src/common/config.h | 2 ++
be/src/olap/rowset/segment_v2/column_reader.cpp | 4 +++-
be/src/olap/rowset/segment_v2/indexed_column_reader.cpp | 3 ++-
4 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 5290acd01cb..984a4ce62d0 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -996,6 +996,8 @@ DEFINE_Bool(enable_file_cache, "false");
// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240},{"path":"/path/to/file_cache2","total_size":21474836480,"query_limit":10737418240}]
DEFINE_String(file_cache_path, "");
DEFINE_Int64(file_cache_each_block_size, "1048576"); // 1MB
+// only cache index pages (prerequisite: enable_file_cache = true)
+DEFINE_Bool(file_cache_index_only, "false");
DEFINE_Bool(clear_file_cache, "false");
DEFINE_Bool(enable_file_cache_query_limit, "false");
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 785cdfe8be5..d3bed036be6 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1042,6 +1042,8 @@ DECLARE_Bool(enable_file_cache);
// format:
[{"path":"/path/to/file_cache","total_size":21474836480,"query_limit":10737418240,"normal_percent":85,
"disposable_percent":10, "index_percent":5}]
DECLARE_String(file_cache_path);
DECLARE_Int64(file_cache_each_block_size);
+// only cache index pages (prerequisite: enable_file_cache = true)
+DECLARE_Bool(file_cache_index_only);
DECLARE_Bool(clear_file_cache);
DECLARE_Bool(enable_file_cache_query_limit);
DECLARE_Int32(file_cache_enter_disk_resource_limit_mode_percent);
diff --git a/be/src/olap/rowset/segment_v2/column_reader.cpp
b/be/src/olap/rowset/segment_v2/column_reader.cpp
index 392917e0d83..7396bce1a55 100644
--- a/be/src/olap/rowset/segment_v2/column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/column_reader.cpp
@@ -342,9 +342,11 @@ Status ColumnReader::read_page(const
ColumnIteratorOptions& iter_opts, const Pag
PageHandle* handle, Slice* page_body,
PageFooterPB* footer,
BlockCompressionCodec* codec) const {
iter_opts.sanity_check();
+ bool use_page_cache = iter_opts.use_page_cache &&
+ (!config::file_cache_index_only || iter_opts.type ==
INDEX_PAGE);
PageReadOptions opts {
.verify_checksum = _opts.verify_checksum,
- .use_page_cache = iter_opts.use_page_cache,
+ .use_page_cache = use_page_cache,
.kept_in_memory = _opts.kept_in_memory,
.type = iter_opts.type,
.file_reader = iter_opts.file_reader,
diff --git a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
index 59251b5595d..94429920877 100644
--- a/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
+++ b/be/src/olap/rowset/segment_v2/indexed_column_reader.cpp
@@ -115,8 +115,9 @@ Status IndexedColumnReader::read_page(const PagePointer&
pp, PageHandle* handle,
PageFooterPB* footer, PageTypePB type,
BlockCompressionCodec* codec, bool
pre_decode) const {
OlapReaderStatistics tmp_stats;
+ bool use_page_cache = _use_page_cache && (!config::file_cache_index_only
|| type == INDEX_PAGE);
PageReadOptions opts {
- .use_page_cache = _use_page_cache,
+ .use_page_cache = use_page_cache,
.kept_in_memory = _kept_in_memory,
.pre_decode = pre_decode,
.type = type,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
