[doris] 29/33: [bug](GC)the issue of incorrect disk usage (#18397)

[morningman]

This is an automated email from the ASF dual-hosted git repository.

morningman pushed a commit to branch doris-for-zhongjin
in repository https://gitbox.apache.org/repos/asf/doris.git

commit 1af97cda039967ec66903fe7d29735da2d2a1ac9
Author: chenlinzhong <490103...@qq.com>
AuthorDate: Sat Apr 8 09:32:36 2023 +0800

    [bug](GC)the issue of incorrect disk usage (#18397)
---
 be/src/io/fs/local_file_system.cpp         | 16 ++++++++++++++++
 be/src/io/fs/local_file_system.h           |  2 ++
 be/src/olap/data_dir.cpp                   |  6 +++---
 be/src/olap/rowset/rowset_meta_manager.cpp |  7 +++++++
 be/src/olap/rowset/rowset_meta_manager.h   |  1 +
 be/src/olap/tablet.cpp                     |  3 ++-
 6 files changed, 31 insertions(+), 4 deletions(-)

diff --git a/be/src/io/fs/local_file_system.cpp 
b/be/src/io/fs/local_file_system.cpp
index 5ffcabca5e..cd08928ba9 100644
--- a/be/src/io/fs/local_file_system.cpp
+++ b/be/src/io/fs/local_file_system.cpp
@@ -115,6 +115,22 @@ Status LocalFileSystem::delete_directory_impl(const Path& 
dir) {
     return Status::OK();
 }
 
+Status LocalFileSystem::delete_directory_or_file(const Path& path) {
+    bool is_dir;
+    Status ret = is_directory(path, &is_dir);
+    if (ret.ok()) {
+        Status s;
+        if (is_dir) {
+            s = delete_directory_impl(path);
+        } else {
+            s = delete_file_impl(path);
+        }
+        return s;
+    } else {
+        return ret;
+    }
+}
+
 Status LocalFileSystem::batch_delete_impl(const std::vector<Path>& files) {
     for (auto& file : files) {
         RETURN_IF_ERROR(delete_file_impl(file));
diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h
index da1704f381..71f17dbe28 100644
--- a/be/src/io/fs/local_file_system.h
+++ b/be/src/io/fs/local_file_system.h
@@ -54,6 +54,8 @@ public:
     Status copy_dirs(const Path& src, const Path& dest);
     // return true if parent path contain sub path
     static bool contain_path(const Path& parent, const Path& sub);
+    // delete dir or file
+    Status delete_directory_or_file(const Path& path);
 
 protected:
     Status create_file_impl(const Path& file, FileWriterPtr* writer) override;
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 3fe2dcd7cc..59a03d5471 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -723,9 +723,9 @@ void DataDir::_process_garbage_path(const std::string& 
path) {
         return;
     }
     if (exists) {
-        LOG(INFO) << "collect garbage dir path: " << path;
-        WARN_IF_ERROR(io::global_local_filesystem()->delete_directory(path),
-                      "remove garbage dir failed");
+        LOG(INFO) << "collect garbage path: " << path;
+        
WARN_IF_ERROR(io::global_local_filesystem()->delete_directory_or_file(path),
+                      "remove garbage failed");
     }
 }
 
diff --git a/be/src/olap/rowset/rowset_meta_manager.cpp 
b/be/src/olap/rowset/rowset_meta_manager.cpp
index 7a58fa9cb6..ac9ae5764e 100644
--- a/be/src/olap/rowset/rowset_meta_manager.cpp
+++ b/be/src/olap/rowset/rowset_meta_manager.cpp
@@ -42,6 +42,13 @@ bool RowsetMetaManager::check_rowset_meta(OlapMeta* meta, 
TabletUid tablet_uid,
     return meta->key_may_exist(META_COLUMN_FAMILY_INDEX, key, &value);
 }
 
+Status RowsetMetaManager::exists(OlapMeta* meta, TabletUid tablet_uid, const 
RowsetId& rowset_id) {
+    std::string key = ROWSET_PREFIX + tablet_uid.to_string() + "_" + 
rowset_id.to_string();
+    std::string value;
+    Status s = meta->get(META_COLUMN_FAMILY_INDEX, key, &value);
+    return s;
+}
+
 Status RowsetMetaManager::get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid,
                                           const RowsetId& rowset_id,
                                           RowsetMetaSharedPtr rowset_meta) {
diff --git a/be/src/olap/rowset/rowset_meta_manager.h 
b/be/src/olap/rowset/rowset_meta_manager.h
index 8c8f3144e0..cca2304ce3 100644
--- a/be/src/olap/rowset/rowset_meta_manager.h
+++ b/be/src/olap/rowset/rowset_meta_manager.h
@@ -31,6 +31,7 @@ namespace doris {
 class RowsetMetaManager {
 public:
     static bool check_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const 
RowsetId& rowset_id);
+    static Status exists(OlapMeta* meta, TabletUid tablet_uid, const RowsetId& 
rowset_id);
 
     static Status get_rowset_meta(OlapMeta* meta, TabletUid tablet_uid, const 
RowsetId& rowset_id,
                                   RowsetMetaSharedPtr rowset_meta);
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index 2bf536468f..8dd0e83c6f 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -1160,7 +1160,8 @@ bool Tablet::check_rowset_id(const RowsetId& rowset_id) {
             return true;
         }
     }
-    if (RowsetMetaManager::check_rowset_meta(_data_dir->get_meta(), 
tablet_uid(), rowset_id)) {
+    Status s = RowsetMetaManager::exists(_data_dir->get_meta(), tablet_uid(), 
rowset_id);
+    if (!s.is<META_KEY_NOT_FOUND>()) {
         return true;
     }
     return false;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org