This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 162b1c5 [Storage] Open data dirs parallelly (#3260)
162b1c5 is described below
commit 162b1c5d8bfd8afb7b01951ea5c5a79d1ffe1679
Author: HuangWei <huangw...@xiaomi.com>
AuthorDate: Tue Apr 7 20:59:56 2020 +0800
[Storage] Open data dirs parallelly (#3260)
---
be/src/olap/rowset/rowset.h | 2 --
be/src/olap/storage_engine.cpp | 45 ++++++++++++++++++++++++++++++++----------
be/src/olap/storage_engine.h | 2 ++
3 files changed, 37 insertions(+), 12 deletions(-)
diff --git a/be/src/olap/rowset/rowset.h b/be/src/olap/rowset/rowset.h
index 30cf314..2c509eb 100644
--- a/be/src/olap/rowset/rowset.h
+++ b/be/src/olap/rowset/rowset.h
@@ -204,8 +204,6 @@ public:
// return whether `path` is one of the files in this rowset
virtual bool check_path(const std::string& path) = 0;
- std::string rowset_path() const { return _rowset_path; }
-
// return an unique identifier string for this rowset
std::string unique_id() const {
return _rowset_path + "/" + rowset_id().to_string();
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index d8be355..16b9025 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -148,16 +148,8 @@ void StorageEngine::load_data_dirs(const
std::vector<DataDir*>& data_dirs) {
OLAPStatus StorageEngine::_open() {
// init store_map
- for (auto& path : _options.store_paths) {
- DataDir* store = new DataDir(path.path, path.capacity_bytes,
path.storage_medium,
- _tablet_manager.get(),
_txn_manager.get());
- auto st = store->init();
- if (!st.ok()) {
- LOG(WARNING) << "Store load failed, path=" << path.path;
- return OLAP_ERR_INVALID_ROOT_PATH;
- }
- _store_map.emplace(path.path, store);
- }
+ RETURN_NOT_OK(_init_store_map());
+
_effective_cluster_id = config::cluster_id;
RETURN_NOT_OK_LOG(_check_all_root_path_cluster_id(), "fail to check
cluster info.");
@@ -185,6 +177,39 @@ OLAPStatus StorageEngine::_open() {
return OLAP_SUCCESS;
}
+OLAPStatus StorageEngine::_init_store_map() {
+ std::vector<DataDir*> tmp_stores;
+ std::vector<std::thread> threads;
+ std::atomic<bool> init_error{false};
+ for (auto& path : _options.store_paths) {
+ DataDir* store = new DataDir(path.path, path.capacity_bytes,
path.storage_medium,
+ _tablet_manager.get(),
_txn_manager.get());
+ tmp_stores.emplace_back(store);
+ threads.emplace_back([store, &init_error]() {
+ auto st = store->init();
+ if (!st.ok()) {
+ init_error = true;
+ LOG(WARNING) << "Store load failed, status="<< st.to_string()
<< ", path=" << store->path();
+ }
+ });
+ }
+ for (auto& thread : threads) {
+ thread.join();
+ }
+
+ if (init_error) {
+ for (auto store : tmp_stores) {
+ delete store;
+ }
+ return OLAP_ERR_INVALID_ROOT_PATH;
+ }
+
+ for (auto store : tmp_stores) {
+ _store_map.emplace(store->path(), store);
+ }
+ return OLAP_SUCCESS;
+}
+
void StorageEngine::_update_storage_medium_type_count() {
set<TStorageMedium::type> available_storage_medium_types;
diff --git a/be/src/olap/storage_engine.h b/be/src/olap/storage_engine.h
index 8d32d9b..7177968 100644
--- a/be/src/olap/storage_engine.h
+++ b/be/src/olap/storage_engine.h
@@ -200,6 +200,8 @@ private:
// Clear status(tables, ...)
void _clear();
+ OLAPStatus _init_store_map();
+
void _update_storage_medium_type_count();
// Some check methods
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
