This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d224624bbe [improvement](session variable)Add enable_file_cache
session variable (#16268)
d224624bbe is described below
commit d224624bbe7e16eabb44cbaed279a8a92d9d36fc
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Wed Feb 1 18:15:03 2023 +0800
[improvement](session variable)Add enable_file_cache session variable
(#16268)
Add enable_file_cache session variable, so that we can close file cache
without restart BE.
---
be/src/io/file_factory.cpp | 3 ++-
be/src/olap/iterators.h | 6 ++++--
be/src/vec/exec/scan/vfile_scanner.cpp | 1 +
docs/en/docs/advanced/variables.md | 4 ++++
docs/zh-CN/docs/advanced/variables.md | 4 ++++
.../main/java/org/apache/doris/qe/SessionVariable.java | 15 +++++++++++++++
gensrc/thrift/PaloInternalService.thrift | 2 ++
7 files changed, 32 insertions(+), 3 deletions(-)
diff --git a/be/src/io/file_factory.cpp b/be/src/io/file_factory.cpp
index f90a2b5489..a970405dd7 100644
--- a/be/src/io/file_factory.cpp
+++ b/be/src/io/file_factory.cpp
@@ -35,6 +35,7 @@
#include "io/local_file_writer.h"
#include "io/s3_reader.h"
#include "io/s3_writer.h"
+#include "olap/iterators.h"
#include "runtime/exec_env.h"
#include "runtime/stream_load/load_stream_mgr.h"
#include "runtime/stream_load/new_load_stream_mgr.h"
@@ -155,7 +156,7 @@ Status FileFactory::create_file_reader(RuntimeProfile*
/*profile*/,
io::FileReaderSPtr* file_reader,
IOContext* io_ctx) {
TFileType::type type = system_properties.system_type;
auto cache_policy = io::FileCachePolicy::NO_CACHE;
- if (config::enable_file_cache) {
+ if (config::enable_file_cache && io_ctx->enable_file_cache) {
cache_policy = io::FileCachePolicy::FILE_BLOCK_CACHE;
}
io::FileReaderOptions reader_options(cache_policy,
io::FileBlockCachePathPolicy());
diff --git a/be/src/olap/iterators.h b/be/src/olap/iterators.h
index db1580d06d..36f765ac88 100644
--- a/be/src/olap/iterators.h
+++ b/be/src/olap/iterators.h
@@ -38,18 +38,20 @@ struct IOContext {
IOContext() = default;
IOContext(const TUniqueId* query_id_, FileCacheStatistics* stats_, bool
is_presistent_,
- bool use_disposable_cache_, bool read_segment_index_)
+ bool use_disposable_cache_, bool read_segment_index_, bool
enable_file_cache)
: query_id(query_id_),
is_persistent(is_presistent_),
use_disposable_cache(use_disposable_cache_),
read_segment_index(read_segment_index_),
- file_cache_stats(stats_) {}
+ file_cache_stats(stats_),
+ enable_file_cache(enable_file_cache) {}
ReaderType reader_type;
const TUniqueId* query_id = nullptr;
bool is_persistent = false;
bool use_disposable_cache = false;
bool read_segment_index = false;
FileCacheStatistics* file_cache_stats = nullptr;
+ bool enable_file_cache = true;
};
namespace vectorized {
struct IteratorRowRef;
diff --git a/be/src/vec/exec/scan/vfile_scanner.cpp
b/be/src/vec/exec/scan/vfile_scanner.cpp
index 90975686fb..0bb61e54c5 100644
--- a/be/src/vec/exec/scan/vfile_scanner.cpp
+++ b/be/src/vec/exec/scan/vfile_scanner.cpp
@@ -77,6 +77,7 @@ Status VFileScanner::prepare(
_io_ctx.reset(new IOContext());
_io_ctx->file_cache_stats = _file_cache_statistics.get();
_io_ctx->query_id = &_state->query_id();
+ _io_ctx->enable_file_cache = _state->query_options().enable_file_cache;
if (vconjunct_ctx_ptr != nullptr) {
// Copy vconjunct_ctx_ptr from scan node to this scanner's
_vconjunct_ctx.
diff --git a/docs/en/docs/advanced/variables.md
b/docs/en/docs/advanced/variables.md
index 050bcfd869..9725672cc3 100644
--- a/docs/en/docs/advanced/variables.md
+++ b/docs/en/docs/advanced/variables.md
@@ -567,3 +567,7 @@ Translated with www.DeepL.com/Translator (free version)
* `group_by_and_having_use_alias_first`
Specifies whether group by and having clauses use column aliases rather
than searching for column name in From clause. The default value is false.
+
+* `enable_file_cache`
+
+ Set wether to use block file cache. This variable takes effect only if the
BE config enable_file_cache=true. The cache is not used when BE config
enable_file_cache=false.
diff --git a/docs/zh-CN/docs/advanced/variables.md
b/docs/zh-CN/docs/advanced/variables.md
index 5c5c1530d2..4c8e8ee06e 100644
--- a/docs/zh-CN/docs/advanced/variables.md
+++ b/docs/zh-CN/docs/advanced/variables.md
@@ -555,4 +555,8 @@ SELECT /*+ SET_VAR(query_timeout = 1,
enable_partition_cache=true) */ sleep(3);
* `group_by_and_having_use_alias_first`
指定group by和having语句是否优先使用列的别名，而非从From语句里寻找列的名字。默认为false。
+
+* `enable_file_cache`
+
+ 控制是否启用block file
cache。该变量只有在be.conf中enable_file_cache=true时才有效，如果be.conf中enable_file_cache=false，则block
file cache一直处于禁用状态。
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index a7cf5db874..e18770e9fd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -259,6 +259,7 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_TWO_PHASE_READ_OPT =
"enable_two_phase_read_opt";
public static final String TWO_PHASE_READ_OPT_LIMIT_THRESHOLD =
"two_phase_read_opt_limit_threshold";
+ public static final String ENABLE_FILE_CACHE = "enable_file_cache";
public static final String GROUP_BY_AND_HAVING_USE_ALIAS_FIRST =
"group_by_and_having_use_alias_first";
@@ -686,6 +687,10 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = GROUP_BY_AND_HAVING_USE_ALIAS_FIRST)
public boolean groupByAndHavingUseAliasFirst = false;
+ // Whether enable block file cache. Only take effect when BE config item
enable_file_cache is true.
+ @VariableMgr.VarAttr(name = ENABLE_FILE_CACHE, needForward = true)
+ public boolean enableFileCache = true;
+
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to
generate some variables,
// not the default value set in the code.
public void initFuzzyModeVariables() {
@@ -1388,6 +1393,14 @@ public class SessionVariable implements Serializable,
Writable {
}
}
+ public boolean isEnableFileCache() {
+ return enableFileCache;
+ }
+
+ public void setEnableFileCache(boolean enableFileCache) {
+ this.enableFileCache = enableFileCache;
+ }
+
/**
* Serialize to thrift object.
* Used for rest api.
@@ -1450,6 +1463,8 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setExternalSortBytesThreshold(externalSortBytesThreshold);
+ tResult.setEnableFileCache(enableFileCache);
+
return tResult;
}
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index 3d51035cf8..65471d8f5c 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -192,6 +192,8 @@ struct TQueryOptions {
58: optional i64 external_sort_bytes_threshold = 0
59: optional i32 partitioned_hash_agg_rows_threshold = 0
+
+ 60: optional bool enable_file_cache = true
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
