github-actions[bot] commented on code in PR #40226:
URL: https://github.com/apache/doris/pull/40226#discussion_r1740878525
##########
be/src/io/cache/fs_file_cache_storage.cpp:
##########
@@ -227,7 +275,7 @@ std::string
FSFileCacheStorage::get_path_in_local_cache(const UInt128Wrapper& va
}
}
-Status FSFileCacheStorage::rebuild_data_structure() const {
+Status FSFileCacheStorage::upgrade_cache_dir_if_necessary() const {
Review Comment:
warning: function 'upgrade_cache_dir_if_necessary' has cognitive complexity
of 119 (threshold 50) [readability-function-cognitive-complexity]
```cpp
Status FSFileCacheStorage::upgrade_cache_dir_if_necessary() const {
^
```
<details>
<summary>Additional context</summary>
**be/src/io/cache/fs_file_cache_storage.cpp:281:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
RETURN_IF_ERROR(read_file_cache_version(&version));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:281:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(read_file_cache_version(&version));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:282:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if (USE_CACHE_VERSION2 && version != "2.0") {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:282:** +1
```cpp
if (USE_CACHE_VERSION2 && version != "2.0") {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:286:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:290:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_it != std::filesystem::directory_iterator(); ++key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:291:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (key_it->is_directory()) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:293:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (cache_key.size() > KEY_PREFIX_LENGTH) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:297:** +5, including nesting
penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(fs->exists(key_prefix, &exists));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:297:** +6, including nesting
penalty of 5, nesting level increased to 6
```cpp
RETURN_IF_ERROR(fs->exists(key_prefix, &exists));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:298:** +5, including nesting
penalty of 4, nesting level increased to 5
```cpp
if (!exists) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:299:** +6, including nesting
penalty of 5, nesting level increased to 6
```cpp
RETURN_IF_ERROR(fs->create_directory(key_prefix));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:299:** +7, including nesting
penalty of 6, nesting level increased to 7
```cpp
RETURN_IF_ERROR(fs->create_directory(key_prefix));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:301:** +5, including nesting
penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(fs->rename(key_it->path(), key_prefix /
cache_key));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:301:** +6, including nesting
penalty of 5, nesting level increased to 6
```cpp
RETURN_IF_ERROR(fs->rename(key_it->path(), key_prefix /
cache_key));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:305:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (!write_file_cache_version().ok()) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:310:** nesting level increased
to 1
```cpp
auto rebuild_dir = [&](std::filesystem::directory_iterator&
upgrade_key_it) -> Status {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:311:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; upgrade_key_it != std::filesystem::directory_iterator();
++upgrade_key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:312:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (upgrade_key_it->path().filename().native().find('_') ==
std::string::npos) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:313:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
RETURN_IF_ERROR(fs->delete_directory(upgrade_key_it->path().native() + "_0"));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:313:** +5, including nesting
penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(fs->delete_directory(upgrade_key_it->path().native() + "_0"));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:314:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
RETURN_IF_ERROR(
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:314:** +5, including nesting
penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:321:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if constexpr (USE_CACHE_VERSION2) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:323:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) [[unlikely]] {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:327:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_prefix_it != std::filesystem::directory_iterator();
++key_prefix_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:328:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (!key_prefix_it->is_directory()) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:332:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (key_prefix_it->path().filename().native().size() !=
KEY_PREFIX_LENGTH) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:335:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
RETURN_IF_ERROR(fs->delete_directory(key_prefix_it->path()));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:335:** +5, including nesting
penalty of 4, nesting level increased to 5
```cpp
RETURN_IF_ERROR(fs->delete_directory(key_prefix_it->path()));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:338:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) [[unlikely]] {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:341:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(rebuild_dir(key_it));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:341:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
RETURN_IF_ERROR(rebuild_dir(key_it));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:343:** +1, nesting level
increased to 1
```cpp
} else {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:345:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) [[unlikely]] {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:348:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
RETURN_IF_ERROR(rebuild_dir(key_it));
^
```
**be/src/common/status.h:628:** expanded from macro 'RETURN_IF_ERROR'
```cpp
do { \
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:348:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
RETURN_IF_ERROR(rebuild_dir(key_it));
^
```
**be/src/common/status.h:630:** expanded from macro 'RETURN_IF_ERROR'
```cpp
if (UNLIKELY(!_status_.ok())) { \
^
```
</details>
##########
be/src/io/cache/fs_file_cache_storage.cpp:
##########
@@ -338,6 +386,72 @@
return Path(_cache_base_path) / "version";
}
+Status FSFileCacheStorage::parse_filename_suffix_to_cache_type(
+ const std::shared_ptr<LocalFileSystem>& fs, const Path& file_path,
long expiration_time,
+ size_t size, size_t* offset, bool* is_tmp, FileCacheType* cache_type)
const {
+ std::error_code ec;
+ std::string offset_with_suffix = file_path.native();
+ auto delim_pos1 = offset_with_suffix.find('_');
+ bool parsed = true;
+
+ try {
+ if (delim_pos1 == std::string::npos) {
+ // same as type "normal"
+ *offset = stoull(offset_with_suffix);
+ } else {
+ *offset = stoull(offset_with_suffix.substr(0, delim_pos1));
+ std::string suffix = offset_with_suffix.substr(delim_pos1 + 1);
+ // not need persistent anymore
+ // if suffix is equals to "tmp", it should be removed too.
+ if (suffix == "tmp") [[unlikely]] {
+ *is_tmp = true;
+ } else {
+ *cache_type = BlockFileCache::string_to_cache_type(suffix);
+ }
+ }
+ } catch (...) {
+ parsed = false;
+ }
+
+ // File in dir with expiration time > 0 should all be TTL type
+ // while expiration time == 0 should all be NORMAL type but
+ // in old days, bug happens, thus break such consistency, e.g.
+ // BEs shut down during cache type transition.
+ // Nowadays, we only use expiration time to decide the type,
+ // i.e. whenever expiration time > 0, it IS TTL, otherwise
+ // it is NORMAL or INDEX depending on its suffix.
+ // From now on, the ttl type encoding in file name is only for
+ // compatibility. It won't be build into the filename, and existing
+ // ones will be ignored.
+ if (expiration_time > 0) {
+ *cache_type = FileCacheType::TTL;
+ } else if (*cache_type == FileCacheType::TTL && expiration_time == 0) {
+ *cache_type = FileCacheType::NORMAL;
+ }
+
+ if (!parsed) {
+ LOG(WARNING) << "parse offset err, path=" << file_path.native();
+ return Status::InternalError("parse offset err, path={}",
file_path.native());
+ }
+ TEST_SYNC_POINT_CALLBACK("BlockFileCache::REMOVE_FILE",
&offset_with_suffix);
+
+ if (ec) {
+ LOG(WARNING) << "failed to file_size: file_name=" << offset_with_suffix
+ << "error=" << ec.message();
+ return Status::InternalError("failed to file_size: file_name={},
error={}",
+ offset_with_suffix, ec.message());
+ }
+
+ if (size == 0 && !(*is_tmp)) {
+ auto st = fs->delete_file(file_path);
+ if (!st.ok()) {
+ LOG_WARNING("delete file {} error", file_path.native()).error(st);
+ }
+ return Status::InternalError("file size is 0, file_name={}",
offset_with_suffix);
+ }
+ return Status::OK();
+}
+
void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr)
const {
Review Comment:
warning: function 'load_cache_info_into_memory' has cognitive complexity of
53 (threshold 50) [readability-function-cognitive-complexity]
```cpp
void FSFileCacheStorage::load_cache_info_into_memory(BlockFileCache* _mgr)
const {
^
```
<details>
<summary>Additional context</summary>
**be/src/io/cache/fs_file_cache_storage.cpp:458:** nesting level increased
to 1
```cpp
auto add_cell_batch_func = [&]() {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:460:** nesting level increased
to 2
```cpp
auto f = [&](const BatchLoadArgs& args) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:462:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (_mgr->_files.contains(args.hash) &&
_mgr->_files[args.hash].contains(args.offset)) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:462:** +1
```cpp
if (_mgr->_files.contains(args.hash) &&
_mgr->_files[args.hash].contains(args.offset)) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:464:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (file_block->expiration_time() !=
args.ctx.expiration_time ||
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:464:** +1
```cpp
if (file_block->expiration_time() !=
args.ctx.expiration_time ||
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:478:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (!args.is_tmp) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:485:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:493:** nesting level increased
to 1
```cpp
auto scan_file_cache = [&](std::filesystem::directory_iterator& key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:495:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_it != std::filesystem::directory_iterator(); ++key_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:504:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) [[unlikely]] {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:513:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
for (; offset_it != std::filesystem::directory_iterator();
++offset_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:518:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (!parse_filename_suffix_to_cache_type(fs,
offset_it->path().filename().native(),
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:535:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (batch_load_buffer.size() >= scan_length) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:543:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if constexpr (USE_CACHE_VERSION2) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:545:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:549:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
for (; key_prefix_it != std::filesystem::directory_iterator();
++key_prefix_it) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:550:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (!key_prefix_it->is_directory()) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:554:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (key_prefix_it->path().filename().native().size() !=
KEY_PREFIX_LENGTH) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:559:** +4, including nesting
penalty of 3, nesting level increased to 4
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:566:** +3, including nesting
penalty of 2, nesting level increased to 3
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:572:** +1, nesting level
increased to 1
```cpp
} else {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:574:** +2, including nesting
penalty of 1, nesting level increased to 2
```cpp
if (ec) {
^
```
**be/src/io/cache/fs_file_cache_storage.cpp:580:** +1, including nesting
penalty of 0, nesting level increased to 1
```cpp
if (!batch_load_buffer.empty()) {
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
