gavinchou commented on code in PR #59269:
URL: https://github.com/apache/doris/pull/59269#discussion_r2645652582
##########
be/src/io/cache/fs_file_cache_storage.cpp:
##########
@@ -970,42 +981,317 @@ FSFileCacheStorage::~FSFileCacheStorage() {
if (_cache_background_load_thread.joinable()) {
_cache_background_load_thread.join();
}
+ stop_leak_cleaner();
}
-size_t FSFileCacheStorage::estimate_file_count_from_statfs() const {
+size_t FSFileCacheStorage::estimate_file_count_from_inode() const {
struct statvfs vfs;
if (statvfs(_cache_base_path.c_str(), &vfs) != 0) {
LOG(WARNING) << "Failed to get filesystem statistics for path: " <<
_cache_base_path
<< ", error: " << strerror(errno);
return 0;
}
- // Get total size of cache directory to estimate file count
- std::error_code ec;
- uintmax_t total_size = 0;
- for (const auto& entry :
std::filesystem::recursive_directory_iterator(_cache_base_path, ec)) {
- if (ec) {
- LOG(WARNING) << "Error accessing directory entry: " <<
ec.message();
- continue;
+ if (vfs.f_files == 0) {
+ LOG(WARNING) << "Filesystem returned zero total inodes for path " <<
_cache_base_path;
+ return 0;
+ }
+
+ size_t inode_used = vfs.f_files - vfs.f_ffree;
+ LOG(INFO) << "Inode usage for cache path " << _cache_base_path << ":
total=" << vfs.f_files
+ << ", free=" << vfs.f_ffree << ", used=" << inode_used;
+ return inode_used;
+}
+
+size_t FSFileCacheStorage::snapshot_metadata_block_count(BlockFileCache* mgr)
const {
+ if (mgr == nullptr) {
+ return 0;
+ }
+ std::lock_guard<std::mutex> lock(mgr->_mutex);
+ size_t total_blocks = 0;
+ for (const auto& [_, blocks_by_offset] : mgr->_files) {
+ total_blocks += blocks_by_offset.size();
+ }
+ return total_blocks;
+}
+
+std::vector<size_t> FSFileCacheStorage::snapshot_metadata_for_hash_offsets(
+ BlockFileCache* mgr, const UInt128Wrapper& hash) const {
+ std::vector<size_t> offsets;
+ if (mgr == nullptr) {
+ return offsets;
+ }
+ std::lock_guard<std::mutex> lock(mgr->_mutex);
+ auto it = mgr->_files.find(hash);
+ if (it == mgr->_files.end()) {
+ return offsets;
+ }
+ offsets.reserve(it->second.size());
+ for (const auto& [offset, _] : it->second) {
+ offsets.push_back(offset);
+ }
+ return offsets;
+}
+
+void FSFileCacheStorage::start_leak_cleaner(BlockFileCache* mgr) {
+ if (config::file_cache_leak_scan_interval_seconds <= 0) {
+ LOG(INFO) << "File cache leak cleaner disabled because interval <= 0";
+ return;
+ }
+ _stop_leak_cleaner.store(false, std::memory_order_relaxed);
+ _cache_leak_cleaner_thread = std::thread([this]() { leak_cleaner_loop();
});
Review Comment:
name every thread
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
