This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new af2d6287611 branch-3.0: [enhancement](cloud) add profile counter for
file cache #45678 (#45892)
af2d6287611 is described below
commit af2d62876115f25d450995b193fab33093f06ae1
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Dec 26 11:09:59 2024 +0800
branch-3.0: [enhancement](cloud) add profile counter for file cache #45678
(#45892)
Cherry-picked from #45678
Co-authored-by: zhengyu <zhangzhen...@selectdb.com>
---
be/src/io/cache/block_file_cache.cpp | 20 +++-
be/src/io/cache/block_file_cache_profile.h | 17 +++
be/src/io/cache/cached_remote_file_reader.cpp | 11 ++
be/src/io/cache/cached_remote_file_reader.h | 9 --
be/src/io/cache/file_cache_common.cpp | 2 +
be/src/io/cache/file_cache_common.h | 16 +++
be/src/io/io_common.h | 5 +
be/test/io/cache/block_file_cache_test.cpp | 146 ++++++++++++++++++++++++++
8 files changed, 214 insertions(+), 12 deletions(-)
diff --git a/be/src/io/cache/block_file_cache.cpp
b/be/src/io/cache/block_file_cache.cpp
index 2a59a5158e4..b5f48d09648 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -40,6 +40,8 @@
#include "io/cache/file_cache_common.h"
#include "io/cache/fs_file_cache_storage.h"
#include "io/cache/mem_file_cache_storage.h"
+#include "util/runtime_profile.h"
+#include "util/stopwatch.hpp"
#include "util/time.h"
#include "vec/common/sip_hash.h"
#include "vec/common/uint128.h"
@@ -770,7 +772,13 @@ FileBlocksHolder BlockFileCache::get_or_set(const
UInt128Wrapper& hash, size_t o
CacheContext& context) {
FileBlock::Range range(offset, offset + size - 1);
- SCOPED_CACHE_LOCK(_mutex);
+ ReadStatistics* stats = context.stats;
+ DCHECK(stats != nullptr);
+ MonotonicStopWatch sw;
+ sw.start();
+ std::lock_guard cache_lock(_mutex);
+ stats->lock_wait_timer += sw.elapsed_time();
+
if (auto iter = _key_to_time.find(hash);
context.cache_type == FileCacheType::INDEX && iter !=
_key_to_time.end()) {
context.cache_type = FileCacheType::TTL;
@@ -778,12 +786,18 @@ FileBlocksHolder BlockFileCache::get_or_set(const
UInt128Wrapper& hash, size_t o
}
/// Get all blocks which intersect with the given range.
- auto file_blocks = get_impl(hash, context, range, cache_lock);
+ FileBlocks file_blocks;
+ {
+ SCOPED_RAW_TIMER(&stats->get_timer);
+ file_blocks = get_impl(hash, context, range, cache_lock);
+ }
if (file_blocks.empty()) {
+ SCOPED_RAW_TIMER(&stats->set_timer);
file_blocks = split_range_into_cells(hash, context, offset, size,
FileBlock::State::EMPTY,
cache_lock);
} else {
+ SCOPED_RAW_TIMER(&stats->set_timer);
fill_holes_with_empty_file_blocks(file_blocks, hash, context, range,
cache_lock);
}
DCHECK(!file_blocks.empty());
@@ -996,7 +1010,6 @@ bool BlockFileCache::try_reserve(const UInt128Wrapper&
hash, const CacheContext&
if (!_async_open_done) {
return try_reserve_during_async_load(size, cache_lock);
}
-
// use this strategy in scenarios where there is insufficient disk
capacity or insufficient number of inodes remaining
// directly eliminate 5 times the size of the space
if (_disk_resource_limit_mode) {
@@ -1055,6 +1068,7 @@ bool BlockFileCache::try_reserve(const UInt128Wrapper&
hash, const CacheContext&
if (cell->releasable()) {
auto& file_block = cell->file_block;
+
std::lock_guard block_lock(file_block->_mutex);
DCHECK(file_block->_download_state ==
FileBlock::State::DOWNLOADED);
to_evict.push_back(cell);
diff --git a/be/src/io/cache/block_file_cache_profile.h
b/be/src/io/cache/block_file_cache_profile.h
index 54118d5094c..f9d9df0939f 100644
--- a/be/src/io/cache/block_file_cache_profile.h
+++ b/be/src/io/cache/block_file_cache_profile.h
@@ -83,6 +83,11 @@ struct FileCacheProfileReporter {
RuntimeProfile::Counter* write_cache_io_timer = nullptr;
RuntimeProfile::Counter* bytes_write_into_cache = nullptr;
RuntimeProfile::Counter* num_skip_cache_io_total = nullptr;
+ RuntimeProfile::Counter* read_cache_file_directly_timer = nullptr;
+ RuntimeProfile::Counter* cache_get_or_set_timer = nullptr;
+ RuntimeProfile::Counter* lock_wait_timer = nullptr;
+ RuntimeProfile::Counter* get_timer = nullptr;
+ RuntimeProfile::Counter* set_timer = nullptr;
FileCacheProfileReporter(RuntimeProfile* profile) {
static const char* cache_profile = "FileCache";
@@ -105,6 +110,13 @@ struct FileCacheProfileReporter {
TUnit::BYTES,
cache_profile, 1);
bytes_scanned_from_remote = ADD_CHILD_COUNTER_WITH_LEVEL(profile,
"BytesScannedFromRemote",
TUnit::BYTES,
cache_profile, 1);
+ read_cache_file_directly_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile,
"ReadCacheFileDirectlyTimer", cache_profile, 1);
+ cache_get_or_set_timer =
+ ADD_CHILD_TIMER_WITH_LEVEL(profile, "CacheGetOrSetTimer",
cache_profile, 1);
+ lock_wait_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "LockWaitTimer",
cache_profile, 1);
+ get_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "GetTimer",
cache_profile, 1);
+ set_timer = ADD_CHILD_TIMER_WITH_LEVEL(profile, "SetTimer",
cache_profile, 1);
}
void update(const FileCacheStatistics* statistics) const {
@@ -119,6 +131,11 @@ struct FileCacheProfileReporter {
COUNTER_UPDATE(num_skip_cache_io_total,
statistics->num_skip_cache_io_total);
COUNTER_UPDATE(bytes_scanned_from_cache,
statistics->bytes_read_from_local);
COUNTER_UPDATE(bytes_scanned_from_remote,
statistics->bytes_read_from_remote);
+ COUNTER_UPDATE(read_cache_file_directly_timer,
statistics->read_cache_file_directly_timer);
+ COUNTER_UPDATE(cache_get_or_set_timer,
statistics->cache_get_or_set_timer);
+ COUNTER_UPDATE(lock_wait_timer, statistics->lock_wait_timer);
+ COUNTER_UPDATE(get_timer, statistics->get_timer);
+ COUNTER_UPDATE(set_timer, statistics->set_timer);
}
};
diff --git a/be/src/io/cache/cached_remote_file_reader.cpp
b/be/src/io/cache/cached_remote_file_reader.cpp
index f16e0019b6d..70765fa707e 100644
--- a/be/src/io/cache/cached_remote_file_reader.cpp
+++ b/be/src/io/cache/cached_remote_file_reader.cpp
@@ -134,6 +134,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset,
Slice result, size_t*
stats.bytes_read += bytes_req;
if (config::enable_read_cache_file_directly) {
// read directly
+ SCOPED_RAW_TIMER(&stats.read_cache_file_directly_timer);
size_t need_read_size = bytes_req;
std::shared_lock lock(_mtx);
if (!_cache_file_readers.empty()) {
@@ -174,8 +175,12 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset,
Slice result, size_t*
// read from cache or remote
auto [align_left, align_size] = s_align_size(offset, bytes_req, size());
CacheContext cache_context(io_ctx);
+ cache_context.stats = &stats;
+ MonotonicStopWatch sw;
+ sw.start();
FileBlocksHolder holder =
_cache->get_or_set(_cache_hash, align_left, align_size,
cache_context);
+ stats.cache_get_or_set_timer += sw.elapsed_time();
std::vector<FileBlockSPtr> empty_blocks;
for (auto& block : holder.file_blocks) {
switch (block->state()) {
@@ -333,6 +338,12 @@ void CachedRemoteFileReader::_update_state(const
ReadStatistics& read_stats,
statis->bytes_write_into_cache += read_stats.bytes_write_into_file_cache;
statis->write_cache_io_timer += read_stats.local_write_timer;
+ statis->read_cache_file_directly_timer +=
read_stats.read_cache_file_directly_timer;
+ statis->cache_get_or_set_timer += read_stats.cache_get_or_set_timer;
+ statis->lock_wait_timer += read_stats.lock_wait_timer;
+ statis->get_timer += read_stats.get_timer;
+ statis->set_timer += read_stats.set_timer;
+
g_skip_cache_num << read_stats.skip_cache;
g_skip_cache_sum << read_stats.skip_cache;
}
diff --git a/be/src/io/cache/cached_remote_file_reader.h
b/be/src/io/cache/cached_remote_file_reader.h
index 685414cfa3a..735e652f94c 100644
--- a/be/src/io/cache/cached_remote_file_reader.h
+++ b/be/src/io/cache/cached_remote_file_reader.h
@@ -67,15 +67,6 @@ private:
std::shared_mutex _mtx;
std::map<size_t, FileBlockSPtr> _cache_file_readers;
- struct ReadStatistics {
- bool hit_cache = true;
- bool skip_cache = false;
- int64_t bytes_read = 0;
- int64_t bytes_write_into_file_cache = 0;
- int64_t remote_read_timer = 0;
- int64_t local_read_timer = 0;
- int64_t local_write_timer = 0;
- };
void _update_state(const ReadStatistics& stats, FileCacheStatistics* state,
bool is_inverted_index) const;
};
diff --git a/be/src/io/cache/file_cache_common.cpp
b/be/src/io/cache/file_cache_common.cpp
index 19041938a08..56525425f75 100644
--- a/be/src/io/cache/file_cache_common.cpp
+++ b/be/src/io/cache/file_cache_common.cpp
@@ -83,6 +83,8 @@ FileBlocksHolderPtr
FileCacheAllocatorBuilder::allocate_cache_holder(size_t offs
ctx.cache_type = _expiration_time == 0 ? FileCacheType::NORMAL :
FileCacheType::TTL;
ctx.expiration_time = _expiration_time;
ctx.is_cold_data = _is_cold_data;
+ ReadStatistics stats;
+ ctx.stats = &stats;
auto holder = _cache->get_or_set(_cache_hash, offset, size, ctx);
return std::make_unique<FileBlocksHolder>(std::move(holder));
}
diff --git a/be/src/io/cache/file_cache_common.h
b/be/src/io/cache/file_cache_common.h
index 0d700d93031..25df07b5ddf 100644
--- a/be/src/io/cache/file_cache_common.h
+++ b/be/src/io/cache/file_cache_common.h
@@ -50,6 +50,21 @@ struct UInt128Wrapper {
bool operator==(const UInt128Wrapper& other) const { return value_ ==
other.value_; }
};
+struct ReadStatistics {
+ bool hit_cache = true;
+ bool skip_cache = false;
+ int64_t bytes_read = 0;
+ int64_t bytes_write_into_file_cache = 0;
+ int64_t remote_read_timer = 0;
+ int64_t local_read_timer = 0;
+ int64_t local_write_timer = 0;
+ int64_t read_cache_file_directly_timer = 0;
+ int64_t cache_get_or_set_timer = 0;
+ int64_t lock_wait_timer = 0;
+ int64_t get_timer = 0;
+ int64_t set_timer = 0;
+};
+
class BlockFileCache;
struct FileBlocksHolder;
using FileBlocksHolderPtr = std::unique_ptr<FileBlocksHolder>;
@@ -134,6 +149,7 @@ struct CacheContext {
FileCacheType cache_type;
int64_t expiration_time {0};
bool is_cold_data {false};
+ ReadStatistics* stats;
};
} // namespace doris::io
diff --git a/be/src/io/io_common.h b/be/src/io/io_common.h
index 4acc0538b7e..be56d0d63ab 100644
--- a/be/src/io/io_common.h
+++ b/be/src/io/io_common.h
@@ -46,6 +46,11 @@ struct FileCacheStatistics {
int64_t write_cache_io_timer = 0;
int64_t bytes_write_into_cache = 0;
int64_t num_skip_cache_io_total = 0;
+ int64_t read_cache_file_directly_timer = 0;
+ int64_t cache_get_or_set_timer = 0;
+ int64_t lock_wait_timer = 0;
+ int64_t get_timer = 0;
+ int64_t set_timer = 0;
};
struct IOContext {
diff --git a/be/test/io/cache/block_file_cache_test.cpp
b/be/test/io/cache/block_file_cache_test.cpp
index 11e99a48052..d8c2dbe384c 100644
--- a/be/test/io/cache/block_file_cache_test.cpp
+++ b/be/test/io/cache/block_file_cache_test.cpp
@@ -256,6 +256,9 @@ void test_file_cache(io::FileCacheType cache_type) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context, other_context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ other_context.stats = &rstats;
context.cache_type = other_context.cache_type = cache_type;
context.query_id = query_id;
other_context.query_id = other_query_id;
@@ -687,6 +690,9 @@ void test_file_cache_memory_storage(io::FileCacheType
cache_type) {
settings.max_query_cache_size = 30;
settings.storage = "memory";
io::CacheContext context, other_context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ other_context.stats = &rstats;
context.cache_type = other_context.cache_type = cache_type;
context.query_id = query_id;
other_context.query_id = other_query_id;
@@ -1117,6 +1123,8 @@ TEST_F(BlockFileCacheTest, max_ttl_size) {
settings.max_file_block_size = 100000;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -1187,6 +1195,8 @@ TEST_F(BlockFileCacheTest,
max_ttl_size_with_other_cache_exist) {
// populate the cache with other cache type
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
context.query_id = query_id;
int64_t offset = 100000000;
@@ -1239,6 +1249,8 @@ TEST_F(BlockFileCacheTest, max_ttl_size_memory_storage) {
settings.max_query_cache_size = 30;
settings.storage = "memory";
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -1299,6 +1311,8 @@ TEST_F(BlockFileCacheTest,
query_limit_heap_use_after_free) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
auto key = io::BlockFileCache::hash("key1");
{
@@ -1384,6 +1398,8 @@ TEST_F(BlockFileCacheTest, query_limit_dcheck) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
auto key = io::BlockFileCache::hash("key1");
{
@@ -1501,6 +1517,8 @@ TEST_F(BlockFileCacheTest, reset_range) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
auto key = io::BlockFileCache::hash("key1");
{
@@ -1551,6 +1569,8 @@ TEST_F(BlockFileCacheTest, change_cache_type) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
auto key = io::BlockFileCache::hash("key1");
{
@@ -1599,6 +1619,8 @@ TEST_F(BlockFileCacheTest,
change_cache_type_memory_storage) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
auto key = io::BlockFileCache::hash("key1");
{
@@ -1643,6 +1665,8 @@ TEST_F(BlockFileCacheTest, fd_cache_remove) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
auto key = io::BlockFileCache::hash("key1");
{
@@ -1725,6 +1749,8 @@ TEST_F(BlockFileCacheTest, fd_cache_evict) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
}
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
auto key = io::BlockFileCache::hash("key1");
config::file_cache_max_file_reader_cache_size = 2;
@@ -1855,6 +1881,10 @@ void
test_file_cache_run_in_resource_limit(io::FileCacheType cache_type) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context, other_context, index_context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ other_context.stats = &rstats;
+ index_context.stats = &rstats;
context.cache_type = other_context.cache_type = cache_type;
context.query_id = query_id;
other_context.query_id = other_query_id;
@@ -1957,6 +1987,9 @@ TEST_F(BlockFileCacheTest, fix_tmp_file) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context, other_context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ other_context.stats = &rstats;
context.cache_type = other_context.cache_type = io::FileCacheType::NORMAL;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
@@ -2015,6 +2048,9 @@ TEST_F(BlockFileCacheTest, test_async_load) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context, other_context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ other_context.stats = &rstats;
context.cache_type = other_context.cache_type = io::FileCacheType::NORMAL;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
@@ -2076,6 +2112,9 @@ TEST_F(BlockFileCacheTest, test_async_load_with_limit) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context, other_context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ other_context.stats = &rstats;
context.cache_type = other_context.cache_type = io::FileCacheType::NORMAL;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
@@ -2144,6 +2183,8 @@ TEST_F(BlockFileCacheTest, ttl_normal) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -2241,6 +2282,8 @@ TEST_F(BlockFileCacheTest, ttl_modify) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -2321,6 +2364,8 @@ TEST_F(BlockFileCacheTest, ttl_modify_memory_storage) {
settings.max_query_cache_size = 30;
settings.storage = "memory";
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -2397,6 +2442,8 @@ TEST_F(BlockFileCacheTest, ttl_change_to_normal) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -2457,6 +2504,8 @@ TEST_F(BlockFileCacheTest,
ttl_change_to_normal_memory_storage) {
settings.max_query_cache_size = 30;
settings.storage = "memory";
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -2513,6 +2562,8 @@ TEST_F(BlockFileCacheTest, ttl_change_expiration_time) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -2584,6 +2635,8 @@ TEST_F(BlockFileCacheTest,
ttl_change_expiration_time_memory_storage) {
settings.max_query_cache_size = 30;
settings.storage = "memory";
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -2640,6 +2693,9 @@ TEST_F(BlockFileCacheTest, io_error) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context, other_context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
+ other_context.stats = &rstats;
context.cache_type = other_context.cache_type = io::FileCacheType::NORMAL;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
@@ -2797,6 +2853,8 @@ TEST_F(BlockFileCacheTest,
remove_directly_when_normal_change_to_ttl) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key1 = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -2855,6 +2913,8 @@ TEST_F(BlockFileCacheTest, recyle_cache_async) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -2924,6 +2984,8 @@ TEST_F(BlockFileCacheTest, recyle_cache_async_ttl) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
auto key2 = io::BlockFileCache::hash("key2");
@@ -3005,6 +3067,8 @@ TEST_F(BlockFileCacheTest, remove_directly) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key1 = io::BlockFileCache::hash("key1");
auto key2 = io::BlockFileCache::hash("key2");
@@ -3087,6 +3151,8 @@ TEST_F(BlockFileCacheTest, test_factory_1) {
EXPECT_EQ(FileCacheFactory::instance()->get_by_path(cache_path3), nullptr);
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
auto cache = FileCacheFactory::instance()->get_by_path(key1);
int i = 0;
while (i++ < 1000) {
@@ -3262,6 +3328,8 @@ TEST_F(BlockFileCacheTest, test_disposable) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = FileCacheType::DISPOSABLE;
auto key = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -3306,6 +3374,8 @@ TEST_F(BlockFileCacheTest, test_query_limit) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 15;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = FileCacheType::NORMAL;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
@@ -3374,6 +3444,8 @@ TEST_F(BlockFileCacheTest, append_many_time) {
settings.max_query_cache_size = 15;
auto key = io::BlockFileCache::hash("key1");
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = FileCacheType::NORMAL;
io::BlockFileCache cache(cache_base_path, settings);
ASSERT_TRUE(cache.initialize());
@@ -3489,6 +3561,8 @@ TEST_F(BlockFileCacheTest, query_file_cache) {
settings.max_query_cache_size = 15;
auto key = io::BlockFileCache::hash("key1");
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = FileCacheType::NORMAL;
{
io::BlockFileCache cache(cache_base_path, settings);
@@ -3556,6 +3630,8 @@ TEST_F(BlockFileCacheTest, query_file_cache_reserve) {
auto key = io::BlockFileCache::hash("key1");
auto key2 = io::BlockFileCache::hash("key2");
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = FileCacheType::NORMAL;
context.query_id = id;
config::enable_file_cache_query_limit = true;
@@ -3635,6 +3711,8 @@ TEST_F(BlockFileCacheTest, cached_remote_file_reader) {
settings.max_file_block_size = 1048576;
settings.max_query_cache_size = 0;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path,
settings).ok());
FileReaderSPtr local_reader;
@@ -3737,6 +3815,8 @@ TEST_F(BlockFileCacheTest,
cached_remote_file_reader_tail) {
settings.max_file_block_size = 1048576;
settings.max_query_cache_size = 0;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path,
settings).ok());
FileReaderSPtr local_reader;
@@ -3799,6 +3879,8 @@ TEST_F(BlockFileCacheTest,
cached_remote_file_reader_error_handle) {
settings.max_file_block_size = 1048576;
settings.max_query_cache_size = 0;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path,
settings).ok());
auto cache = FileCacheFactory::instance()->_caches[0].get();
@@ -3877,6 +3959,8 @@ TEST_F(BlockFileCacheTest,
cached_remote_file_reader_init) {
settings.max_file_block_size = 1048576;
settings.max_query_cache_size = 0;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path,
settings).ok());
FileReaderSPtr local_reader;
@@ -3928,6 +4012,8 @@ TEST_F(BlockFileCacheTest,
cached_remote_file_reader_concurrent) {
settings.max_file_block_size = 1048576;
settings.max_query_cache_size = 0;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path,
settings).ok());
FileReaderSPtr local_reader;
@@ -4000,6 +4086,8 @@ TEST_F(BlockFileCacheTest,
cached_remote_file_reader_concurrent_2) {
settings.max_file_block_size = 1048576;
settings.max_query_cache_size = 0;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path,
settings).ok());
FileReaderSPtr local_reader;
@@ -4068,6 +4156,8 @@ TEST_F(BlockFileCacheTest, test_hot_data) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
auto key1 = io::BlockFileCache::hash("key1");
auto key2 = io::BlockFileCache::hash("key2");
int64_t expiration_time = UnixSeconds() + 300;
@@ -4152,6 +4242,8 @@ TEST_F(BlockFileCacheTest,
test_async_load_with_error_file_1) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::INDEX;
auto key = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -4225,6 +4317,8 @@ TEST_F(BlockFileCacheTest,
test_async_load_with_error_file_2) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::INDEX;
auto key = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -4461,6 +4555,8 @@ TEST_F(BlockFileCacheTest,
remove_if_cached_when_isnt_releasable) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
@@ -4502,6 +4598,8 @@ TEST_F(BlockFileCacheTest,
cached_remote_file_reader_opt_lock) {
settings.max_file_block_size = 1048576;
settings.max_query_cache_size = 0;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
ASSERT_TRUE(FileCacheFactory::instance()->create_file_cache(cache_base_path,
settings).ok());
io::FileReaderOptions opts;
@@ -4609,6 +4707,8 @@ TEST_F(BlockFileCacheTest, remove_from_other_queue_1) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -4680,6 +4780,8 @@ TEST_F(BlockFileCacheTest, remove_from_other_queue_2) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -4786,6 +4888,8 @@ TEST_F(BlockFileCacheTest, recyle_unvalid_ttl_async) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
io::BlockFileCache cache(cache_base_path, settings);
@@ -4850,6 +4954,8 @@ TEST_F(BlockFileCacheTest, reset_capacity) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
auto key = io::BlockFileCache::hash("key1");
auto key2 = io::BlockFileCache::hash("key2");
@@ -4926,6 +5032,8 @@ TEST_F(BlockFileCacheTest, change_cache_type1) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -5000,6 +5108,8 @@ TEST_F(BlockFileCacheTest, change_cache_type2) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
context.cache_type = io::FileCacheType::NORMAL;
@@ -5181,6 +5291,8 @@ TEST_F(BlockFileCacheTest, test_load) {
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.expiration_time = expiration;
auto key = io::BlockFileCache::hash("key1");
@@ -5309,6 +5421,8 @@ TEST_F(BlockFileCacheTest,
populate_empty_cache_with_disposable) {
size_t limit = 1000000;
size_t cache_max = 10000000;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::DISPOSABLE;
context.query_id = query_id;
// int64_t cur_time = UnixSeconds();
@@ -5414,6 +5528,8 @@ TEST_F(BlockFileCacheTest,
populate_empty_cache_with_normal) {
size_t limit = 3000000;
size_t cache_max = 10000000;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::NORMAL;
context.query_id = query_id;
// int64_t cur_time = UnixSeconds();
@@ -5518,6 +5634,8 @@ TEST_F(BlockFileCacheTest,
populate_empty_cache_with_index) {
size_t limit = 1000000;
size_t cache_max = 10000000;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::INDEX;
context.query_id = query_id;
// int64_t cur_time = UnixSeconds();
@@ -5622,6 +5740,8 @@ TEST_F(BlockFileCacheTest, populate_empty_cache_with_ttl)
{
size_t limit = 5000000;
size_t cache_max = 10000000;
io::CacheContext context;
+ ReadStatistics rstats;
+ context.stats = &rstats;
context.cache_type = io::FileCacheType::TTL;
context.query_id = query_id;
int64_t cur_time = UnixSeconds();
@@ -5738,6 +5858,8 @@ TEST_F(BlockFileCacheTest, disposable_seize_after_normal)
{
size_t cache_max = 10000000;
io::CacheContext context1;
+ ReadStatistics rstats;
+ context1.stats = &rstats;
context1.cache_type = io::FileCacheType::NORMAL;
context1.query_id = query_id;
auto key1 = io::BlockFileCache::hash("key1");
@@ -5764,6 +5886,7 @@ TEST_F(BlockFileCacheTest, disposable_seize_after_normal)
{
ASSERT_EQ(cache.get_stats_unsafe()["normal_queue_curr_size"], cache_max);
// our hero comes to the stage
io::CacheContext context2;
+ context2.stats = &rstats;
context2.cache_type = io::FileCacheType::DISPOSABLE;
context2.query_id = query_id;
auto key2 = io::BlockFileCache::hash("key2");
@@ -5894,6 +6017,8 @@ TEST_F(BlockFileCacheTest, seize_after_full) {
size_t cache_max = 10000000;
io::CacheContext context1;
+ ReadStatistics rstats;
+ context1.stats = &rstats;
context1.cache_type = args.first_type;
context1.query_id = query_id;
if (args.first_type == io::FileCacheType::TTL) {
@@ -5921,6 +6046,7 @@ TEST_F(BlockFileCacheTest, seize_after_full) {
ASSERT_EQ(cache.get_stats_unsafe()[args.first_metrics], cache_max);
// our hero comes to the stage
io::CacheContext context2;
+ context2.stats = &rstats;
context2.cache_type = args.second_type;
context2.query_id = query_id;
if (context2.cache_type == io::FileCacheType::TTL) {
@@ -6009,6 +6135,8 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_disposable) {
ASSERT_TRUE(cache.get_async_open_success());
io::CacheContext context1;
+ ReadStatistics rstats;
+ context1.stats = &rstats;
context1.cache_type = io::FileCacheType::NORMAL;
context1.query_id = query_id;
auto key1 = io::BlockFileCache::hash("key1");
@@ -6030,6 +6158,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_disposable) {
blocks.clear();
}
io::CacheContext context2;
+ context2.stats = &rstats;
context2.cache_type = io::FileCacheType::INDEX;
context2.query_id = query_id;
auto key2 = io::BlockFileCache::hash("key2");
@@ -6051,6 +6180,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_disposable) {
blocks.clear();
}
io::CacheContext context3;
+ context3.stats = &rstats;
context3.cache_type = io::FileCacheType::TTL;
context3.query_id = query_id;
context3.expiration_time = UnixSeconds() + 120;
@@ -6079,6 +6209,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_disposable) {
// our hero comes to the stage
io::CacheContext context4;
+ context4.stats = &rstats;
context4.cache_type = io::FileCacheType::DISPOSABLE;
context4.query_id = query_id;
auto key4 = io::BlockFileCache::hash("key4");
@@ -6183,6 +6314,8 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_normal) {
ASSERT_TRUE(cache.get_async_open_success());
io::CacheContext context1;
+ ReadStatistics rstats;
+ context1.stats = &rstats;
context1.cache_type = io::FileCacheType::DISPOSABLE;
context1.query_id = query_id;
auto key1 = io::BlockFileCache::hash("key1");
@@ -6204,6 +6337,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_normal) {
blocks.clear();
}
io::CacheContext context2;
+ context2.stats = &rstats;
context2.cache_type = io::FileCacheType::INDEX;
context2.query_id = query_id;
auto key2 = io::BlockFileCache::hash("key2");
@@ -6225,6 +6359,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_normal) {
blocks.clear();
}
io::CacheContext context3;
+ context3.stats = &rstats;
context3.cache_type = io::FileCacheType::TTL;
context3.query_id = query_id;
context3.expiration_time = UnixSeconds() + 120;
@@ -6253,6 +6388,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_normal) {
// our hero comes to the stage
io::CacheContext context4;
+ context4.stats = &rstats;
context4.cache_type = io::FileCacheType::NORMAL;
context4.query_id = query_id;
auto key4 = io::BlockFileCache::hash("key4");
@@ -6356,6 +6492,8 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_index) {
ASSERT_TRUE(cache.get_async_open_success());
io::CacheContext context1;
+ ReadStatistics rstats;
+ context1.stats = &rstats;
context1.cache_type = io::FileCacheType::DISPOSABLE;
context1.query_id = query_id;
auto key1 = io::BlockFileCache::hash("key1");
@@ -6377,6 +6515,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_index) {
blocks.clear();
}
io::CacheContext context2;
+ context2.stats = &rstats;
context2.cache_type = io::FileCacheType::NORMAL;
context2.query_id = query_id;
auto key2 = io::BlockFileCache::hash("key2");
@@ -6398,6 +6537,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_index) {
blocks.clear();
}
io::CacheContext context3;
+ context3.stats = &rstats;
context3.cache_type = io::FileCacheType::TTL;
context3.query_id = query_id;
context3.expiration_time = UnixSeconds() + 120;
@@ -6426,6 +6566,7 @@ TEST_F(BlockFileCacheTest,
evict_privilege_order_for_index) {
// our hero comes to the stage
io::CacheContext context4;
+ context4.stats = &rstats;
context4.cache_type = io::FileCacheType::INDEX;
context4.query_id = query_id;
auto key4 = io::BlockFileCache::hash("key4");
@@ -6529,6 +6670,8 @@ TEST_F(BlockFileCacheTest, evict_privilege_order_for_ttl)
{
ASSERT_TRUE(cache.get_async_open_success());
io::CacheContext context1;
+ ReadStatistics rstats;
+ context1.stats = &rstats;
context1.cache_type = io::FileCacheType::DISPOSABLE;
context1.query_id = query_id;
auto key1 = io::BlockFileCache::hash("key1");
@@ -6550,6 +6693,7 @@ TEST_F(BlockFileCacheTest, evict_privilege_order_for_ttl)
{
blocks.clear();
}
io::CacheContext context2;
+ context2.stats = &rstats;
context2.cache_type = io::FileCacheType::INDEX;
context2.query_id = query_id;
auto key2 = io::BlockFileCache::hash("key2");
@@ -6571,6 +6715,7 @@ TEST_F(BlockFileCacheTest, evict_privilege_order_for_ttl)
{
blocks.clear();
}
io::CacheContext context3;
+ context3.stats = &rstats;
context3.cache_type = io::FileCacheType::NORMAL;
context3.query_id = query_id;
auto key3 = io::BlockFileCache::hash("key3");
@@ -6598,6 +6743,7 @@ TEST_F(BlockFileCacheTest, evict_privilege_order_for_ttl)
{
// our hero comes to the stage
io::CacheContext context4;
+ context4.stats = &rstats;
context4.cache_type = io::FileCacheType::TTL;
context4.query_id = query_id;
context4.expiration_time = UnixSeconds() + 120;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
