This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new d110629 Optimize performance of TxnManager::build_expire_txn_map
(#3269)
d110629 is described below
commit d110629a5f7c582d7b047863e2228cecd7ead3f9
Author: Dayue Gao <gaoda...@meituan.com>
AuthorDate: Tue Apr 7 23:54:05 2020 +0800
Optimize performance of TxnManager::build_expire_txn_map (#3269)
It's not possible to insert duplicated transaction ids for a specific
tablet, therefore we could use map<TabletInfo, vector<int64_t>> instead of
map<TabletInfo, set<int64_t>> for expire_txn_map.
---
be/src/olap/tablet_manager.cpp | 12 +++++-------
be/src/olap/txn_manager.cpp | 15 ++++++++-------
be/src/olap/txn_manager.h | 2 +-
3 files changed, 14 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 14382bb..793fd45 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -918,8 +918,9 @@ OLAPStatus
TabletManager::report_all_tablets_info(std::map<TTabletId, TTablet>*
LOG(INFO) << "begin to report all tablets info";
// build the expired txn map first, outside the tablet map lock
- std::map<TabletInfo, std::set<int64_t>> expire_txn_map;
+ std::map<TabletInfo, std::vector<int64_t>> expire_txn_map;
StorageEngine::instance()->txn_manager()->build_expire_txn_map(&expire_txn_map);
+ LOG(INFO) << "find expired transactions for " << expire_txn_map.size() <<
" tablets";
DorisMetrics::report_all_tablets_requests_total.increment(1);
@@ -937,16 +938,13 @@ OLAPStatus
TabletManager::report_all_tablets_info(std::map<TTabletId, TTablet>*
TTabletInfo tablet_info;
tablet_ptr->build_tablet_report_info(&tablet_info);
- // find expire transaction corresponding to this tablet
+ // find expired transaction corresponding to this tablet
TabletInfo tinfo(tablet_id, tablet_ptr->schema_hash(),
tablet_ptr->tablet_uid());
- vector<int64_t> transaction_ids;
auto find = expire_txn_map.find(tinfo);
if (find != expire_txn_map.end()) {
- for(auto& it : find->second) {
- transaction_ids.push_back(it);
- }
+ tablet_info.__set_transaction_ids(find->second);
+ expire_txn_map.erase(find);
}
- tablet_info.__set_transaction_ids(transaction_ids);
t_tablet.tablet_infos.push_back(tablet_info);
}
diff --git a/be/src/olap/txn_manager.cpp b/be/src/olap/txn_manager.cpp
index 11eebba..f152164 100755
--- a/be/src/olap/txn_manager.cpp
+++ b/be/src/olap/txn_manager.cpp
@@ -493,24 +493,25 @@ bool TxnManager::has_txn(TPartitionId partition_id,
TTransactionId transaction_i
return found;
}
-void TxnManager::build_expire_txn_map(std::map<TabletInfo, std::set<int64_t>>*
expire_txn_map) {
+void TxnManager::build_expire_txn_map(std::map<TabletInfo,
std::vector<int64_t>>* expire_txn_map) {
time_t now = time(nullptr);
- int64_t counter = 0;
// traverse the txn map, and get all expired txns
ReadLock txn_rdlock(&_txn_map_lock);
for (auto& it : _txn_tablet_map) {
+ auto txn_id = it.first.second;
for (auto& t_map : it.second) {
double diff = difftime(now, t_map.second.creation_time);
if (diff >= config::pending_data_expire_time_sec) {
- if (expire_txn_map->find(t_map.first) ==
expire_txn_map->end()) {
- (*expire_txn_map)[t_map.first] = std::set<int64_t>();
+ (*expire_txn_map)[t_map.first].push_back(txn_id);
+ if (VLOG_IS_ON(3)) {
+ VLOG(3) << "find expired txn."
+ << " tablet=" << t_map.first.to_string()
+ << " transaction_id=" << txn_id
+ << " exist_sec=" << diff;
}
- (*expire_txn_map)[t_map.first].insert(it.first.second);
- counter++;
}
}
}
- LOG(INFO) << "get " << counter << " expired txns";
}
void TxnManager::get_partition_ids(const TTransactionId transaction_id,
std::vector<TPartitionId>* partition_ids) {
diff --git a/be/src/olap/txn_manager.h b/be/src/olap/txn_manager.h
index c51e253..d4a3e92 100755
--- a/be/src/olap/txn_manager.h
+++ b/be/src/olap/txn_manager.h
@@ -133,7 +133,7 @@ public:
// get all expired txns and save tham in expire_txn_map.
// This is currently called before reporting all tablet info, to avoid
iterating txn map for every tablets.
- void build_expire_txn_map(std::map<TabletInfo, std::set<int64_t>>*
expire_txn_map);
+ void build_expire_txn_map(std::map<TabletInfo, std::vector<int64_t>>*
expire_txn_map);
void force_rollback_tablet_related_txns(OlapMeta* meta, TTabletId
tablet_id, SchemaHash schema_hash, TabletUid tablet_uid);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
