xiaokang commented on code in PR #16003:
URL: https://github.com/apache/doris/pull/16003#discussion_r1071954964
##########
be/src/common/config.h:
##########
@@ -875,6 +875,16 @@ CONF_String(disposable_file_cache_path, "");
CONF_Int64(file_cache_max_file_segment_size, "4194304"); // 4MB
CONF_Bool(clear_file_cache, "false");
CONF_Bool(enable_file_cache_query_limit, "false");
+
+// inverted index searcher cache
+// cache entry stay time after lookup, default 1h
+CONF_mInt32(index_cache_entry_stay_time_after_lookup_s, "3600");
+// inverted index searcher cache size
+CONF_String(inverted_index_searcher_cache_limit, "10%");
+// set `true` to enable insert searcher into cache when write inverted index
data
+CONF_Bool(enable_write_index_searcher_cache, "true");
+CONF_Bool(enable_index_cache_check_timestamp, "true");
+
Review Comment:
use inverted_index_ prefix for all inverted index related config
##########
be/src/olap/lru_cache.h:
##########
@@ -300,6 +302,11 @@ class HandleTable {
void _resize();
};
+typedef std::priority_queue<std::pair<int64_t, LRUHandle*>,
Review Comment:
add comment
##########
be/src/olap/storage_engine.cpp:
##########
@@ -46,6 +46,7 @@
#include "olap/push_handler.h"
#include "olap/reader.h"
#include "olap/rowset/rowset_meta_manager.h"
+#include "olap/rowset/segment_v2/inverted_index_cache.h"
Review Comment:
unused include?
##########
be/src/olap/lru_cache.h:
##########
@@ -353,12 +364,20 @@ class LRUCache {
uint64_t _lookup_count = 0; // cache查找总次数
uint64_t _hit_count = 0; // 命中cache的总次数
+
+ CacheValueExtractor _cache_value_extractor;
+ bool _cache_value_check_timestamp = false;
+ LRUHandleHeap _sort_normal_entries_with_timestamp;
Review Comment:
sort -> sorted?
##########
be/src/olap/lru_cache.cpp:
##########
@@ -240,6 +240,16 @@ void LRUCache::release(Cache::Handle* handle) {
} else if (e->priority == CachePriority::DURABLE) {
_lru_append(&_lru_durable, e);
}
+
+ if (_cache_value_check_timestamp) {
Review Comment:
add some comment
##########
be/src/olap/lru_cache.h:
##########
@@ -146,6 +147,7 @@ class CacheKey {
enum class CachePriority { NORMAL = 0, DURABLE = 1 };
using CacheValuePredicate = std::function<bool(const void*)>;
+using CacheValueExtractor = std::function<int64_t(const void*)>;
Review Comment:
add some comment
##########
be/src/olap/lru_cache.cpp:
##########
@@ -250,6 +260,40 @@ void LRUCache::release(Cache::Handle* handle) {
}
}
+void LRUCache::_evict_from_lru_with_time(size_t total_size, LRUHandle**
to_remove_head) {
+ // 1. evict normal cache entries
+ while (_usage + total_size > _capacity &&
!_sort_normal_entries_with_timestamp.empty()) {
+ auto entry_pair = _sort_normal_entries_with_timestamp.top();
+ LRUHandle* remove_handle = entry_pair.second;
+ if (_cache_value_extractor(remove_handle->value) != entry_pair.first) {
Review Comment:
add comment for this check
##########
be/src/olap/lru_cache.h:
##########
@@ -146,6 +147,7 @@ class CacheKey {
enum class CachePriority { NORMAL = 0, DURABLE = 1 };
using CacheValuePredicate = std::function<bool(const void*)>;
+using CacheValueExtractor = std::function<int64_t(const void*)>;
Review Comment:
Is CacheTimeExtractor more intuitive?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
