This is an automated email from the ASF dual-hosted git repository.
zouxinyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ce1b39e79d [fix](profile) avoid unnecessary refresh profile of
TabletsChannel
ce1b39e79d is described below
commit ce1b39e79dbe72b63de189553daa64b53413242c
Author: Xinyi Zou <zouxiny...@gmail.com>
AuthorDate: Tue Jun 20 21:09:43 2023 +0800
[fix](profile) avoid unnecessary refresh profile of TabletsChannel
Before, refresh the TabletsChannel profile in the LoadChannelMgr refresh
memory statistics thread
This means that enable_profile=false will refresh and have performance loss
in stress test
---
be/src/runtime/load_channel.cpp | 7 +++++++
be/src/runtime/tablets_channel.cpp | 15 ++++++++++++---
be/src/runtime/tablets_channel.h | 6 ++----
3 files changed, 21 insertions(+), 7 deletions(-)
diff --git a/be/src/runtime/load_channel.cpp b/be/src/runtime/load_channel.cpp
index 24701c04b3..925ef17308 100644
--- a/be/src/runtime/load_channel.cpp
+++ b/be/src/runtime/load_channel.cpp
@@ -189,6 +189,13 @@ void
LoadChannel::_report_profile(PTabletWriterAddBlockResult* response) {
// and ensures to update the latest LoadChannel profile according to the
timestamp.
_self_profile->set_timestamp(_last_updated_time);
+ {
+ std::lock_guard<SpinLock> l(_tablets_channels_lock);
+ for (auto& it : _tablets_channels) {
+ it.second->refresh_profile();
+ }
+ }
+
TRuntimeProfileTree tprofile;
_profile->to_thrift(&tprofile);
ThriftSerializer ser(false, 4096);
diff --git a/be/src/runtime/tablets_channel.cpp
b/be/src/runtime/tablets_channel.cpp
index 05b17c5931..fe0465a780 100644
--- a/be/src/runtime/tablets_channel.cpp
+++ b/be/src/runtime/tablets_channel.cpp
@@ -253,6 +253,18 @@ void TabletsChannel::_close_wait(DeltaWriter* writer,
}
int64_t TabletsChannel::mem_consumption() {
+ int64_t mem_usage = 0;
+ {
+ std::lock_guard<SpinLock> l(_tablet_writers_lock);
+ for (auto& it : _tablet_writers) {
+ int64_t writer_mem = it.second->mem_consumption(MemType::ALL);
+ mem_usage += writer_mem;
+ }
+ }
+ return mem_usage;
+}
+
+void TabletsChannel::refresh_profile() {
int64_t write_mem_usage = 0;
int64_t flush_mem_usage = 0;
int64_t max_tablet_mem_usage = 0;
@@ -260,7 +272,6 @@ int64_t TabletsChannel::mem_consumption() {
int64_t max_tablet_flush_mem_usage = 0;
{
std::lock_guard<SpinLock> l(_tablet_writers_lock);
- _mem_consumptions.clear();
for (auto& it : _tablet_writers) {
int64_t write_mem = it.second->mem_consumption(MemType::WRITE);
write_mem_usage += write_mem;
@@ -270,7 +281,6 @@ int64_t TabletsChannel::mem_consumption() {
if (flush_mem > max_tablet_flush_mem_usage)
max_tablet_flush_mem_usage = flush_mem;
if (write_mem + flush_mem > max_tablet_mem_usage)
max_tablet_mem_usage = write_mem + flush_mem;
- _mem_consumptions.emplace(write_mem + flush_mem, it.first);
}
}
COUNTER_SET(_memory_usage_counter, write_mem_usage + flush_mem_usage);
@@ -279,7 +289,6 @@ int64_t TabletsChannel::mem_consumption() {
COUNTER_SET(_max_tablet_memory_usage_counter, max_tablet_mem_usage);
COUNTER_SET(_max_tablet_write_memory_usage_counter,
max_tablet_write_mem_usage);
COUNTER_SET(_max_tablet_flush_memory_usage_counter,
max_tablet_flush_mem_usage);
- return write_mem_usage + flush_mem_usage;
}
void TabletsChannel::get_active_memtable_mem_consumption(
diff --git a/be/src/runtime/tablets_channel.h b/be/src/runtime/tablets_channel.h
index b9235925b0..4e523a1acf 100644
--- a/be/src/runtime/tablets_channel.h
+++ b/be/src/runtime/tablets_channel.h
@@ -115,6 +115,8 @@ public:
int64_t mem_consumption();
+ void refresh_profile();
+
void get_active_memtable_mem_consumption(
std::multimap<int64_t, int64_t, std::greater<int64_t>>*
mem_consumptions);
@@ -195,10 +197,6 @@ private:
bool _write_single_replica = false;
- // mem -> tablet_id
- // sort by memory size
- std::multimap<int64_t, int64_t, std::greater<int64_t>> _mem_consumptions;
-
RuntimeProfile* _profile;
RuntimeProfile::Counter* _add_batch_number_counter = nullptr;
RuntimeProfile::HighWaterMarkCounter* _memory_usage_counter = nullptr;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
