This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new dbece745f45 [fix](memory) Fix prune all LRU Cache based on number
#34601
dbece745f45 is described below
commit dbece745f45145b68ebf27cb1d0c3ec39bad125a
Author: Xinyi Zou <zouxiny...@gmail.com>
AuthorDate: Sun May 12 05:28:42 2024 +0800
[fix](memory) Fix prune all LRU Cache based on number #34601
---
be/src/runtime/memory/cache_policy.h | 1 +
be/src/runtime/memory/lru_cache_policy.h | 46 +++++++++++++++++++++++---------
2 files changed, 34 insertions(+), 13 deletions(-)
diff --git a/be/src/runtime/memory/cache_policy.h
b/be/src/runtime/memory/cache_policy.h
index 7333fe4c937..75bf4881894 100644
--- a/be/src/runtime/memory/cache_policy.h
+++ b/be/src/runtime/memory/cache_policy.h
@@ -24,6 +24,7 @@
namespace doris {
static constexpr int32_t CACHE_MIN_FREE_SIZE = 67108864; // 64M
+static constexpr int32_t CACHE_MIN_FREE_NUMBER = 1024;
// Base of all caches. register to CacheManager when cache is constructed.
class CachePolicy {
diff --git a/be/src/runtime/memory/lru_cache_policy.h
b/be/src/runtime/memory/lru_cache_policy.h
index 8f0806e9da9..773817393c7 100644
--- a/be/src/runtime/memory/lru_cache_policy.h
+++ b/be/src/runtime/memory/lru_cache_policy.h
@@ -124,7 +124,10 @@ public:
uint64_t new_id() { return _cache->new_id(); };
// Subclass can override this method to determine whether to do the minor
or full gc
- virtual bool exceed_prune_limit() { return mem_consumption() >
CACHE_MIN_FREE_SIZE; }
+ virtual bool exceed_prune_limit() {
+ return _lru_cache_type == LRUCacheType::SIZE ? mem_consumption() >
CACHE_MIN_FREE_SIZE
+ : get_usage() >
CACHE_MIN_FREE_NUMBER;
+ }
// Try to prune the cache if expired.
void prune_stale() override {
@@ -142,8 +145,8 @@ public:
curtime);
};
- LOG(INFO) << fmt::format("[MemoryGC] {} prune stale start,
consumption {}",
- type_string(_type), mem_consumption());
+ LOG(INFO) << fmt::format("[MemoryGC] {} prune stale start,
consumption {}, usage {}",
+ type_string(_type), mem_consumption(),
get_usage());
// Prune cache in lazy mode to save cpu and minimize the time
holding write lock
PrunedInfo pruned_info = _cache->prune_if(pred, true);
COUNTER_SET(_freed_entrys_counter, pruned_info.pruned_count);
@@ -154,10 +157,19 @@ public:
type_string(_type), _freed_entrys_counter->value(),
_freed_memory_counter->value(),
_prune_stale_number_counter->value());
} else {
- LOG(INFO) << fmt::format(
- "[MemoryGC] {} not need prune stale, consumption {} less
than "
- "CACHE_MIN_FREE_SIZE {}",
- type_string(_type), mem_consumption(),
CACHE_MIN_FREE_SIZE);
+ if (_lru_cache_type == LRUCacheType::SIZE) {
+ LOG(INFO) << fmt::format(
+ "[MemoryGC] {} not need prune stale,
LRUCacheType::SIZE consumption {} "
+ "less "
+ "than CACHE_MIN_FREE_SIZE {}",
+ type_string(_type), mem_consumption(),
CACHE_MIN_FREE_SIZE);
+ } else if (_lru_cache_type == LRUCacheType::NUMBER) {
+ LOG(INFO) << fmt::format(
+ "[MemoryGC] {} not need prune stale,
LRUCacheType::NUMBER usage {} less "
+ "than "
+ "CACHE_MIN_FREE_NUMBER {}",
+ type_string(_type), get_usage(),
CACHE_MIN_FREE_NUMBER);
+ }
}
}
@@ -170,8 +182,8 @@ public:
if ((force && mem_consumption() != 0) || exceed_prune_limit()) {
COUNTER_SET(_cost_timer, (int64_t)0);
SCOPED_TIMER(_cost_timer);
- LOG(INFO) << fmt::format("[MemoryGC] {} prune all start,
consumption {}",
- type_string(_type), mem_consumption());
+ LOG(INFO) << fmt::format("[MemoryGC] {} prune all start,
consumption {}, usage {}",
+ type_string(_type), mem_consumption(),
get_usage());
PrunedInfo pruned_info = _cache->prune();
COUNTER_SET(_freed_entrys_counter, pruned_info.pruned_count);
COUNTER_SET(_freed_memory_counter, pruned_info.pruned_size);
@@ -181,10 +193,18 @@ public:
type_string(_type), _freed_entrys_counter->value(),
_freed_memory_counter->value(),
_prune_all_number_counter->value(), force);
} else {
- LOG(INFO) << fmt::format(
- "[MemoryGC] {} not need prune all, force is {},
consumption {}, "
- "CACHE_MIN_FREE_SIZE {}",
- type_string(_type), force, mem_consumption(),
CACHE_MIN_FREE_SIZE);
+ if (_lru_cache_type == LRUCacheType::SIZE) {
+ LOG(INFO) << fmt::format(
+ "[MemoryGC] {} not need prune all, force is {},
LRUCacheType::SIZE "
+ "consumption {}, "
+ "CACHE_MIN_FREE_SIZE {}",
+ type_string(_type), force, mem_consumption(),
CACHE_MIN_FREE_SIZE);
+ } else if (_lru_cache_type == LRUCacheType::NUMBER) {
+ LOG(INFO) << fmt::format(
+ "[MemoryGC] {} not need prune all, force is {},
LRUCacheType::NUMBER "
+ "usage {}, CACHE_MIN_FREE_NUMBER {}",
+ type_string(_type), force, get_usage(),
CACHE_MIN_FREE_NUMBER);
+ }
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
