[incubator-doris] branch master updated: StorageEngine: unused_rowsets use unordered_multimap (#3207)

[lichaoyong]

This is an automated email from the ASF dual-hosted git repository.

lichaoyong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 0462607  StorageEngine: unused_rowsets use unordered_multimap (#3207)
0462607 is described below

commit 0462607d8d537afea73b0f31e31d598cfe1c04c1
Author: HuangWei <huangw...@xiaomi.com>
AuthorDate: Fri Mar 27 14:30:31 2020 +0800

    StorageEngine: unused_rowsets use unordered_multimap (#3207)
---
 be/src/olap/rowset/rowset.h    |  2 ++
 be/src/olap/storage_engine.cpp | 39 +++++++++++++++++++--------------------
 be/src/olap/storage_engine.h   |  1 +
 3 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 2c509eb..30cf314 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -204,6 +204,8 @@ public:
     // return whether `path` is one of the files in this rowset
     virtual bool check_path(const std::string& path) = 0;
 
+    std::string rowset_path() const { return _rowset_path; }
+
     // return an unique identifier string for this rowset
     std::string unique_id() const {
         return _rowset_path + "/" + rowset_id().to_string();
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 9290807..d8be355 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -713,36 +713,41 @@ void StorageEngine::_parse_default_rowset_type() {
 }
 
 void StorageEngine::start_delete_unused_rowset() {
-    _gc_mutex.lock();
+    MutexLock lock(&_gc_mutex);
     for (auto it = _unused_rowsets.begin(); it != _unused_rowsets.end();) {
         if (it->second.use_count() != 1) {
             ++it;
         } else if (it->second->need_delete_file()) {
             VLOG(3) << "start to remove rowset:" << it->second->rowset_id()
-                    << ", version:" << it->second->version().first << "-" << 
it->second->version().second;
+                    << ", version:" << it->second->version().first << "-"
+                    << it->second->version().second;
             OLAPStatus status = it->second->remove();
-            VLOG(3) << "remove rowset:" << it->second->rowset_id() << " 
finished. status:" << status;
+            VLOG(3) << "remove rowset:" << it->second->rowset_id()
+                    << " finished. status:" << status;
             it = _unused_rowsets.erase(it);
         }
     }
-    _gc_mutex.unlock();
 }
 
 void StorageEngine::add_unused_rowset(RowsetSharedPtr rowset) {
-    if (rowset == nullptr) { return; }
-    _gc_mutex.lock();
+    if (rowset == nullptr) {
+        return;
+    }
+
     VLOG(3) << "add unused rowset, rowset id:" << rowset->rowset_id()
-            << ", version:" << rowset->version().first
-            << "-" << rowset->version().second
+            << ", version:" << rowset->version().first << "-" << 
rowset->version().second
             << ", unique id:" << rowset->unique_id();
-    auto it = _unused_rowsets.find(rowset->unique_id());
+
+    auto rowset_id = rowset->rowset_id().to_string();
+
+    MutexLock lock(&_gc_mutex);
+    auto it = _unused_rowsets.find(rowset_id);
     if (it == _unused_rowsets.end()) {
         rowset->set_need_delete_file();
         rowset->close();
-        _unused_rowsets[rowset->unique_id()] = rowset;
+        _unused_rowsets[rowset_id] = rowset;
         release_rowset_id(rowset->rowset_id());
     }
-    _gc_mutex.unlock();
 }
 
 // TODO(zc): refactor this funciton
@@ -918,15 +923,9 @@ OLAPStatus StorageEngine::execute_task(EngineTask* task) {
 
 // check whether any unused rowsets's id equal to rowset_id
 bool StorageEngine::check_rowset_id_in_unused_rowsets(const RowsetId& 
rowset_id) {
-    _gc_mutex.lock();
-    for (auto& _unused_rowset_pair : _unused_rowsets) {
-        if (_unused_rowset_pair.second->rowset_id() == rowset_id) {
-            _gc_mutex.unlock();
-            return true;
-        }
-    }
-    _gc_mutex.unlock();
-    return false;
+    MutexLock lock(&_gc_mutex);
+    auto search = _unused_rowsets.find(rowset_id.to_string());
+    return search != _unused_rowsets.end();
 }
 
 }  // namespace doris
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 3cb5dd0..8d32d9b 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -296,6 +296,7 @@ private:
     static StorageEngine* _s_instance;
 
     Mutex _gc_mutex;
+    // map<rowset_id(str), RowsetSharedPtr>, if we use RowsetId as the key, we 
need custom hash func
     std::unordered_map<std::string, RowsetSharedPtr> _unused_rowsets;
 
     bool _stop_bg_worker = false;


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org