gavinchou commented on code in PR #59269:
URL: https://github.com/apache/doris/pull/59269#discussion_r2681075473
##########
be/src/io/cache/fs_file_cache_storage.cpp:
##########
@@ -996,82 +1063,632 @@ FSFileCacheStorage::~FSFileCacheStorage() {
if (_cache_background_load_thread.joinable()) {
_cache_background_load_thread.join();
}
+ stop_leak_cleaner();
}
-size_t FSFileCacheStorage::estimate_file_count_from_statfs() const {
- struct statvfs vfs;
- if (statvfs(_cache_base_path.c_str(), &vfs) != 0) {
- LOG(WARNING) << "Failed to get filesystem statistics for path: " <<
_cache_base_path
- << ", error: " << strerror(errno);
+size_t FSFileCacheStorage::estimate_file_count_from_inode() const {
+ int64_t duration_ns = 0;
+ size_t cache_files = 0;
+ {
+ SCOPED_RAW_TIMER(&duration_ns);
+ do {
+ struct statvfs vfs {};
+ int statvfs_res = 0;
+#ifdef BE_TEST
+ if (auto* hooks = inode_test_hooks(); hooks &&
hooks->statvfs_override) {
+ statvfs_res = hooks->statvfs_override(_cache_base_path, &vfs);
+ } else
+#endif
+ {
+ statvfs_res = statvfs(_cache_base_path.c_str(), &vfs);
+ }
+ if (statvfs_res != 0) {
+ LOG(WARNING) << "Failed to get filesystem statistics for path:
" << _cache_base_path
+ << ", error: " << strerror(errno);
+ break;
+ }
+
+ if (vfs.f_files == 0) {
+ LOG(WARNING) << "Filesystem returned zero total inodes for
path "
+ << _cache_base_path;
+ break;
+ }
+
+ struct stat cache_stat {};
+ int lstat_res = 0;
+#ifdef BE_TEST
+ if (auto* hooks = inode_test_hooks(); hooks &&
hooks->lstat_override) {
+ lstat_res = hooks->lstat_override(_cache_base_path,
&cache_stat);
+ } else
+#endif
+ {
+ lstat_res = lstat(_cache_base_path.c_str(), &cache_stat);
+ }
+ if (lstat_res != 0) {
+ LOG(WARNING) << "Failed to stat cache base path " <<
_cache_base_path << ": "
+ << strerror(errno);
+ break;
+ }
+
+ size_t total_inodes_used = vfs.f_files - vfs.f_ffree;
+ size_t non_cache_inodes = estimate_non_cache_inode_usage();
+ size_t directory_inodes = estimate_cache_directory_inode_usage();
+
+ if (total_inodes_used > non_cache_inodes + directory_inodes) {
+ cache_files = total_inodes_used - non_cache_inodes -
directory_inodes;
+ } else {
+ LOG(WARNING) << fmt::format(
+ "Inode subtraction underflow: total={} non_cache={}
directory={}",
+ total_inodes_used, non_cache_inodes, directory_inodes);
+ }
+
+ LOG(INFO) << fmt::format(
+ "Cache inode estimation: total_used={}, non_cache={},
directories≈{}, files≈{}",
+ total_inodes_used, non_cache_inodes, directory_inodes,
cache_files);
+ } while (false);
+ }
+ const double duration_ms = static_cast<double>(duration_ns) / 1'000'000.0;
+ LOG(INFO) << fmt::format("estimate_file_count_from_inode
duration_ms={:.3f}, files={}",
Review Comment:
LOG all of them
```
size_t total_inodes_used = vfs.f_files - vfs.f_ffree;
size_t non_cache_inodes = estimate_non_cache_inode_usage();
size_t directory_inodes = estimate_cache_directory_inode_usage();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
