This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 2fc60740e2 [branch-1.2](profile) add free block memory profile (#19668)
2fc60740e2 is described below
commit 2fc60740e2dba5a517f9cf32c58e915dcfe133df
Author: Xinyi Zou <zouxiny...@gmail.com>
AuthorDate: Wed May 17 08:54:34 2023 +0800
[branch-1.2](profile) add free block memory profile (#19668)
---
be/src/vec/exec/scan/scanner_context.cpp | 7 +++++++
be/src/vec/exec/scan/scanner_context.h | 2 ++
be/src/vec/exec/scan/vscan_node.cpp | 4 ++++
be/src/vec/exec/scan/vscan_node.h | 2 ++
4 files changed, 15 insertions(+)
diff --git a/be/src/vec/exec/scan/scanner_context.cpp
b/be/src/vec/exec/scan/scanner_context.cpp
index 952007532e..81da875d53 100644
--- a/be/src/vec/exec/scan/scanner_context.cpp
+++ b/be/src/vec/exec/scan/scanner_context.cpp
@@ -37,6 +37,8 @@ Status ScannerContext::init() {
_max_thread_num = _max_thread_num == 0 ? 1 : _max_thread_num;
DCHECK(_max_thread_num > 0);
+ _free_blocks_memory_usage = _parent->_free_blocks_memory_usage;
+
// 2. Calculate how many blocks need to be preallocated.
// The calculation logic is as follows:
// 1. Assuming that at most M rows can be scanned in one
scan(config::doris_scanner_row_num),
@@ -53,10 +55,13 @@ Status ScannerContext::init() {
// The free blocks is used for final output block of scanners.
// So use _output_tuple_desc;
+ int64_t free_blocks_memory_usage = 0;
for (int i = 0; i < pre_alloc_block_count; ++i) {
auto block = new vectorized::Block(_output_tuple_desc->slots(),
real_block_size);
+ free_blocks_memory_usage += block->allocated_bytes();
_free_blocks.emplace_back(block);
}
+ _free_blocks_memory_usage->add(free_blocks_memory_usage);
#ifndef BE_TEST
// 3. get thread token
@@ -81,6 +86,7 @@ vectorized::Block* ScannerContext::get_free_block(bool*
get_free_block) {
if (!_free_blocks.empty()) {
auto block = _free_blocks.back();
_free_blocks.pop_back();
+ _free_blocks_memory_usage->add(-block->allocated_bytes());
return block;
}
}
@@ -92,6 +98,7 @@ vectorized::Block* ScannerContext::get_free_block(bool*
get_free_block) {
void ScannerContext::return_free_block(vectorized::Block* block) {
block->clear_column_data();
+ _free_blocks_memory_usage->add(block->allocated_bytes());
std::lock_guard<std::mutex> l(_free_blocks_lock);
_free_blocks.emplace_back(block);
}
diff --git a/be/src/vec/exec/scan/scanner_context.h
b/be/src/vec/exec/scan/scanner_context.h
index 24a490673d..f4c4a297d0 100644
--- a/be/src/vec/exec/scan/scanner_context.h
+++ b/be/src/vec/exec/scan/scanner_context.h
@@ -213,6 +213,8 @@ private:
int64_t _num_ctx_scheduling = 0;
int64_t _num_scanner_scheduling = 0;
+
+ RuntimeProfile::HighWaterMarkCounter* _free_blocks_memory_usage = nullptr;
};
} // namespace vectorized
} // namespace doris
diff --git a/be/src/vec/exec/scan/vscan_node.cpp
b/be/src/vec/exec/scan/vscan_node.cpp
index ce80396fa6..002229ebb3 100644
--- a/be/src/vec/exec/scan/vscan_node.cpp
+++ b/be/src/vec/exec/scan/vscan_node.cpp
@@ -158,6 +158,10 @@ Status VScanNode::_init_profile() {
_scanner_profile.reset(new RuntimeProfile("VScanner"));
runtime_profile()->add_child(_scanner_profile.get(), true, nullptr);
+ auto* memory_usage = _scanner_profile->create_child("PeakMemoryUsage",
true, true);
+ _runtime_profile->add_child(memory_usage, false, nullptr);
+ _free_blocks_memory_usage =
memory_usage->AddHighWaterMarkCounter("FreeBlocks", TUnit::BYTES);
+
_scan_timer = ADD_TIMER(_scanner_profile, "ScannerGetBlockTime");
_scan_cpu_timer = ADD_TIMER(_scanner_profile, "ScannerCpuTime");
_prefilter_timer = ADD_TIMER(_scanner_profile, "ScannerPrefilterTime");
diff --git a/be/src/vec/exec/scan/vscan_node.h
b/be/src/vec/exec/scan/vscan_node.h
index 251c5dd50f..0653b55ead 100644
--- a/be/src/vec/exec/scan/vscan_node.h
+++ b/be/src/vec/exec/scan/vscan_node.h
@@ -259,6 +259,8 @@ protected:
// Max num of scanner thread
RuntimeProfile::Counter* _max_scanner_thread_num = nullptr;
+ RuntimeProfile::HighWaterMarkCounter* _free_blocks_memory_usage;
+
private:
// Register and get all runtime filters at Init phase.
Status _register_runtime_filter();
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
