This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 62ac2559e92 (cloud-merge) Fix index data cannot be writed into ttl dir
(#37485)
62ac2559e92 is described below
commit 62ac2559e92ce731e52d5c31cd07e1b659711510
Author: Lightman <31928846+lchangli...@users.noreply.github.com>
AuthorDate: Thu Jul 18 22:40:23 2024 +0800
(cloud-merge) Fix index data cannot be writed into ttl dir (#37485)
When write index data in ttl dir, it will cannot find the dir because
expiration_time is 0. Reset context.expiration_time if the file is ttl
file.
---
be/src/io/cache/block_file_cache.cpp | 7 ++++++-
be/src/io/cache/block_file_cache.h | 2 +-
be/src/io/cache/cached_remote_file_reader.cpp | 2 +-
be/test/io/cache/block_file_cache_test.cpp | 16 ++++++++++++++--
4 files changed, 22 insertions(+), 5 deletions(-)
diff --git a/be/src/io/cache/block_file_cache.cpp
b/be/src/io/cache/block_file_cache.cpp
index 6c320a21bb3..a2ec9f82c86 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -637,10 +637,15 @@ void
BlockFileCache::fill_holes_with_empty_file_blocks(FileBlocks& file_blocks,
}
FileBlocksHolder BlockFileCache::get_or_set(const UInt128Wrapper& hash, size_t
offset, size_t size,
- const CacheContext& context) {
+ CacheContext& context) {
FileBlock::Range range(offset, offset + size - 1);
std::lock_guard cache_lock(_mutex);
+ if (auto iter = _key_to_time.find(hash);
+ context.cache_type == FileCacheType::INDEX && iter !=
_key_to_time.end()) {
+ context.cache_type = FileCacheType::TTL;
+ context.expiration_time = iter->second;
+ }
/// Get all blocks which intersect with the given range.
auto file_blocks = get_impl(hash, context, range, cache_lock);
diff --git a/be/src/io/cache/block_file_cache.h
b/be/src/io/cache/block_file_cache.h
index e7de154432e..c9668b236c8 100644
--- a/be/src/io/cache/block_file_cache.h
+++ b/be/src/io/cache/block_file_cache.h
@@ -86,7 +86,7 @@ public:
* it is guaranteed that these file blocks are not removed from cache.
*/
FileBlocksHolder get_or_set(const UInt128Wrapper& hash, size_t offset,
size_t size,
- const CacheContext& context);
+ CacheContext& context);
/**
* Clear all cached data for this cache instance async
diff --git a/be/src/io/cache/cached_remote_file_reader.cpp
b/be/src/io/cache/cached_remote_file_reader.cpp
index d4b4157388a..0a46c98390e 100644
--- a/be/src/io/cache/cached_remote_file_reader.cpp
+++ b/be/src/io/cache/cached_remote_file_reader.cpp
@@ -224,7 +224,7 @@ Status CachedRemoteFileReader::read_at_impl(size_t offset,
Slice result, size_t*
st = block->finalize();
}
if (!st.ok()) {
- LOG_WARNING("Write data to file cache failed").error(st);
+ LOG_EVERY_N(WARNING, 100) << "Write data to file cache failed.
err=" << st.msg();
} else {
_insert_file_reader(block);
}
diff --git a/be/test/io/cache/block_file_cache_test.cpp
b/be/test/io/cache/block_file_cache_test.cpp
index d6e2cf6053e..7cbc54d095a 100644
--- a/be/test/io/cache/block_file_cache_test.cpp
+++ b/be/test/io/cache/block_file_cache_test.cpp
@@ -1555,9 +1555,9 @@ TEST_F(BlockFileCacheTest, ttl_normal) {
query_id.hi = 1;
query_id.lo = 1;
io::FileCacheSettings settings;
- settings.query_queue_size = 30;
+ settings.query_queue_size = 50;
settings.query_queue_elements = 5;
- settings.capacity = 30;
+ settings.capacity = 50;
settings.max_file_block_size = 30;
settings.max_query_cache_size = 30;
io::CacheContext context;
@@ -1596,6 +1596,18 @@ TEST_F(BlockFileCacheTest, ttl_normal) {
assert_range(1, blocks[0], io::FileBlock::Range(50, 59),
io::FileBlock::State::DOWNLOADED);
EXPECT_EQ(blocks[0]->cache_type(), io::FileCacheType::TTL);
}
+ {
+ context.cache_type = io::FileCacheType::INDEX;
+ context.expiration_time = 0;
+ auto holder = cache.get_or_set(key2, 60, 10, context); /// Add range
[60, 69]
+ auto blocks = fromHolder(holder);
+ ASSERT_EQ(blocks.size(), 1);
+ assert_range(1, blocks[0], io::FileBlock::Range(60, 69),
io::FileBlock::State::EMPTY);
+ ASSERT_TRUE(blocks[0]->get_or_set_downloader() ==
io::FileBlock::get_caller_id());
+ download(blocks[0]);
+ assert_range(1, blocks[0], io::FileBlock::Range(60, 69),
io::FileBlock::State::DOWNLOADED);
+ EXPECT_EQ(blocks[0]->cache_type(), io::FileCacheType::TTL);
+ }
{
cache.modify_expiration_time(key2, modify_time);
context.expiration_time = modify_time;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
