This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new f625bbb06ea [enhancement](cloud) fix TTL cache size negative overflow
(#42869)
f625bbb06ea is described below
commit f625bbb06ea856e8b28c50729d9c911d9663943b
Author: zhengyu <freeman.zhang1...@gmail.com>
AuthorDate: Tue Oct 29 23:11:08 2024 +0800
[enhancement](cloud) fix TTL cache size negative overflow (#42869)
reset_range enlarges the size if the download size is larger than the
original size. TTL LRU queue missed that while still subtracting the
larger size when entries were removed from the queue, causing the size
metric negative overflow.
Pick #41987
Signed-off-by: freemandealer <freeman.zhang1...@gmail.com>
---
be/src/io/cache/block_file_cache.cpp | 25 ++++++++++---------------
1 file changed, 10 insertions(+), 15 deletions(-)
diff --git a/be/src/io/cache/block_file_cache.cpp
b/be/src/io/cache/block_file_cache.cpp
index 321d9d73d89..998b31ff493 100644
--- a/be/src/io/cache/block_file_cache.cpp
+++ b/be/src/io/cache/block_file_cache.cpp
@@ -243,15 +243,12 @@ void BlockFileCache::use_cell(const FileBlockCell& cell,
FileBlocks* result, boo
result->push_back(cell.file_block);
}
- if (cell.file_block->cache_type() != FileCacheType::TTL ||
- config::enable_ttl_cache_evict_using_lru) {
- auto& queue = get_queue(cell.file_block->cache_type());
- DCHECK(cell.queue_iterator) << "impossible";
- /// Move to the end of the queue. The iterator remains valid.
- if (move_iter_flag) {
- queue.move_to_end(*cell.queue_iterator, cache_lock);
- }
+ auto& queue = get_queue(cell.file_block->cache_type());
+ /// Move to the end of the queue. The iterator remains valid.
+ if (cell.queue_iterator && move_iter_flag) {
+ queue.move_to_end(*cell.queue_iterator, cache_lock);
}
+
cell.update_atime();
cell.is_deleted = false;
}
@@ -357,7 +354,7 @@ FileBlocks BlockFileCache::get_impl(const UInt128Wrapper&
hash, const CacheConte
auto st =
cell.file_block->change_cache_type_between_ttl_and_others(
FileCacheType::NORMAL);
if (st.ok()) {
- if (config::enable_ttl_cache_evict_using_lru) {
+ if (cell.queue_iterator) {
auto& ttl_queue = get_queue(FileCacheType::TTL);
ttl_queue.remove(cell.queue_iterator.value(),
cache_lock);
}
@@ -1052,7 +1049,7 @@ bool BlockFileCache::remove_if_ttl_file_unlock(const
UInt128Wrapper& file_key, b
auto st =
cell.file_block->change_cache_type_between_ttl_and_others(
FileCacheType::NORMAL);
if (st.ok()) {
- if (config::enable_ttl_cache_evict_using_lru) {
+ if (cell.queue_iterator) {
ttl_queue.remove(cell.queue_iterator.value(),
cache_lock);
}
auto& queue = get_queue(FileCacheType::NORMAL);
@@ -1129,7 +1126,7 @@ void BlockFileCache::reset_range(const UInt128Wrapper&
hash, size_t offset, size
_files.find(hash)->second.find(offset) !=
_files.find(hash)->second.end());
FileBlockCell* cell = get_cell(hash, offset, cache_lock);
DCHECK(cell != nullptr);
- if (cell->file_block->cache_type() != FileCacheType::TTL) {
+ if (cell->queue_iterator) {
auto& queue = get_queue(cell->file_block->cache_type());
DCHECK(queue.contains(hash, offset, cache_lock));
auto iter = queue.get(hash, offset, cache_lock);
@@ -1805,10 +1802,8 @@ std::string BlockFileCache::clear_file_cache_directly() {
<< " time_elapsed=" << duration_cast<milliseconds>(steady_clock::now()
- start).count()
<< " num_files=" << num_files << " cache_size=" << cache_size
<< " index_queue_size=" << index_queue_size << " normal_queue_size=" <<
normal_queue_size
- << " disposible_queue_size=" << disposible_queue_size;
- if (config::enable_ttl_cache_evict_using_lru) {
- ss << "ttl_queue_size=" << ttl_queue_size;
- }
+ << " disposible_queue_size=" << disposible_queue_size <<
"ttl_queue_size=" << ttl_queue_size;
+
auto msg = ss.str();
LOG(INFO) << msg;
return msg;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
