dataroaring commented on code in PR #49456:
URL: https://github.com/apache/doris/pull/49456#discussion_r2011918850
##########
be/src/io/cache/block_file_cache.cpp:
##########
@@ -1872,6 +1877,72 @@ void BlockFileCache::run_background_gc() {
}
}
+void BlockFileCache::run_background_lru_dump() {
+ int64_t interval_time_seconds = 60; // Dump every minute
+ while (!_close) {
+ {
+ std::unique_lock close_lock(_close_mtx);
+ _close_cv.wait_for(close_lock,
std::chrono::seconds(interval_time_seconds));
+ if (_close) {
+ break;
+ }
+ }
+
+ // Dump each queue
+ auto dump_queue = [&](LRUQueue& queue, const std::string& queue_name) {
+ std::vector<std::tuple<UInt128Wrapper, size_t, size_t>> elements;
+
+ // Acquire mutex and copy elements
+ {
+ SCOPED_CACHE_LOCK(_mutex, this);
+ size_t count = 0;
+ for (const auto& [hash, offset, size] : queue) {
+ if (count++ >= 5000) break;
+ elements.emplace_back(hash, offset, size);
+ }
+ }
+
+ // Write to disk
+ std::string filename = fmt::format("{}/lru_dump_{}.bin",
_cache_base_path, queue_name);
+ std::ofstream out(filename, std::ios::binary);
+ if (out) {
+ for (const auto& [hash, offset, size] : elements) {
+ out.write(reinterpret_cast<const char*>(&hash),
sizeof(hash));
+ out.write(reinterpret_cast<const char*>(&offset),
sizeof(offset));
+ out.write(reinterpret_cast<const char*>(&size),
sizeof(size));
+ }
+ }
+ };
+
+ dump_queue(_disposable_queue, "disposable");
+ dump_queue(_index_queue, "index");
+ dump_queue(_normal_queue, "normal");
+ dump_queue(_ttl_queue, "ttl");
+ }
+}
+
+void BlockFileCache::restore_lru_queues_from_disk() {
+ auto restore_queue = [&](LRUQueue& queue, const std::string& queue_name) {
+ std::string filename = fmt::format("{}/lru_dump_{}.bin",
_cache_base_path, queue_name);
+ std::ifstream in(filename, std::ios::binary);
+ if (in) {
+ UInt128Wrapper hash;
+ size_t offset, size;
+ while (in.read(reinterpret_cast<char*>(&hash), sizeof(hash)) &&
+ in.read(reinterpret_cast<char*>(&offset), sizeof(offset)) &&
+ in.read(reinterpret_cast<char*>(&size), sizeof(size))) {
Review Comment:
We'd better use pb, then we can handle upgrade easily.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
