This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5c1113b8021 [opt](scanner profile) Rename some filed name to keep
consistent with audit log. (#41993)
5c1113b8021 is described below
commit 5c1113b8021429fbd5277fffcd5f4e0195f44f57
Author: zhiqiang <seuhezhiqi...@163.com>
AuthorDate: Sat Oct 19 18:03:04 2024 +0800
[opt](scanner profile) Rename some filed name to keep consistent with audit
log. (#41993)
---
be/src/pipeline/exec/olap_scan_operator.cpp | 1 -
be/src/pipeline/exec/olap_scan_operator.h | 1 -
be/src/pipeline/exec/scan_operator.cpp | 7 +++++++
be/src/pipeline/exec/scan_operator.h | 3 +++
be/src/vec/exec/scan/new_olap_scanner.cpp | 24 +++++++++++++-----------
be/src/vec/exec/scan/new_olap_scanner.h | 4 ++--
6 files changed, 25 insertions(+), 15 deletions(-)
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index 2696e426786..0d1cb362ea0 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -59,7 +59,6 @@ Status OlapScanLocalState::_init_profile() {
_block_fetch_timer = ADD_TIMER(_scanner_profile, "BlockFetchTime");
_delete_bitmap_get_agg_timer = ADD_TIMER(_scanner_profile,
"DeleteBitmapGetAggTime");
_sync_rowset_timer = ADD_TIMER(_scanner_profile, "SyncRowsetTime");
- _raw_rows_counter = ADD_COUNTER(_segment_profile, "RawRowsRead",
TUnit::UNIT);
_block_convert_timer = ADD_TIMER(_scanner_profile, "BlockConvertTime");
_block_init_timer = ADD_TIMER(_segment_profile, "BlockInitTime");
_block_init_seek_timer = ADD_TIMER(_segment_profile, "BlockInitSeekTime");
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index 4aa9a232225..c972c7ce99a 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -110,7 +110,6 @@ private:
RuntimeProfile::Counter* _read_compressed_counter = nullptr;
RuntimeProfile::Counter* _decompressor_timer = nullptr;
RuntimeProfile::Counter* _read_uncompressed_counter = nullptr;
- RuntimeProfile::Counter* _raw_rows_counter = nullptr;
RuntimeProfile::Counter* _rows_vec_cond_filtered_counter = nullptr;
RuntimeProfile::Counter* _rows_short_circuit_cond_filtered_counter =
nullptr;
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index 081b0b9625b..4f3c97bab71 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -1064,6 +1064,13 @@ Status ScanLocalState<Derived>::_init_profile() {
_peak_running_scanner =
_scanner_profile->AddHighWaterMarkCounter("PeakRunningScanner",
TUnit::UNIT);
+
+ // Rows read from storage.
+ // Include the rows read from doris page cache.
+ _scan_rows = ADD_COUNTER(_runtime_profile, "ScanRows", TUnit::UNIT);
+ // Size of data that read from storage.
+ // Does not include rows that are cached by doris page cache.
+ _scan_bytes = ADD_COUNTER(_runtime_profile, "ScanBytes", TUnit::BYTES);
return Status::OK();
}
diff --git a/be/src/pipeline/exec/scan_operator.h
b/be/src/pipeline/exec/scan_operator.h
index 2b0d82ee788..bf650cb8495 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -128,6 +128,9 @@ protected:
RuntimeProfile::Counter* _num_scanners = nullptr;
RuntimeProfile::Counter* _wait_for_rf_timer = nullptr;
+
+ RuntimeProfile::Counter* _scan_rows = nullptr;
+ RuntimeProfile::Counter* _scan_bytes = nullptr;
};
template <typename LocalStateType>
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index a1d3c71f573..60240618655 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -540,14 +540,15 @@ Status NewOlapScanner::close(RuntimeState* state) {
void NewOlapScanner::_update_realtime_counters() {
pipeline::OlapScanLocalState* local_state =
static_cast<pipeline::OlapScanLocalState*>(_local_state);
- auto& stats = _tablet_reader->stats();
+ const OlapReaderStatistics& stats = _tablet_reader->stats();
COUNTER_UPDATE(local_state->_read_compressed_counter,
stats.compressed_bytes_read);
- _compressed_bytes_read += stats.compressed_bytes_read;
+ COUNTER_UPDATE(local_state->_scan_bytes, stats.compressed_bytes_read);
+ _scan_bytes += stats.compressed_bytes_read;
_tablet_reader->mutable_stats()->compressed_bytes_read = 0;
- COUNTER_UPDATE(local_state->_raw_rows_counter, stats.raw_rows_read);
+ COUNTER_UPDATE(local_state->_scan_rows, stats.raw_rows_read);
+ _scan_rows += stats.raw_rows_read;
// if raw_rows_read is reset, scanNode will scan all table rows which may
cause BE crash
- _raw_rows_read += stats.raw_rows_read;
_tablet_reader->mutable_stats()->raw_rows_read = 0;
}
@@ -564,7 +565,8 @@ void NewOlapScanner::_collect_profile_before_close() {
#define INCR_COUNTER(Parent)
\
COUNTER_UPDATE(Parent->_io_timer, stats.io_ns);
\
COUNTER_UPDATE(Parent->_read_compressed_counter,
stats.compressed_bytes_read); \
- _compressed_bytes_read += stats.compressed_bytes_read;
\
+ COUNTER_UPDATE(Parent->_scan_bytes, stats.compressed_bytes_read);
\
+ _scan_bytes += stats.compressed_bytes_read;
\
COUNTER_UPDATE(Parent->_decompressor_timer, stats.decompress_ns);
\
COUNTER_UPDATE(Parent->_read_uncompressed_counter,
stats.uncompressed_bytes_read); \
COUNTER_UPDATE(Parent->_block_load_timer, stats.block_load_ns);
\
@@ -572,8 +574,8 @@ void NewOlapScanner::_collect_profile_before_close() {
COUNTER_UPDATE(Parent->_block_fetch_timer, stats.block_fetch_ns);
\
COUNTER_UPDATE(Parent->_delete_bitmap_get_agg_timer,
stats.delete_bitmap_get_agg_ns); \
COUNTER_UPDATE(Parent->_block_convert_timer, stats.block_convert_ns);
\
- COUNTER_UPDATE(Parent->_raw_rows_counter, stats.raw_rows_read);
\
- _raw_rows_read += _tablet_reader->mutable_stats()->raw_rows_read;
\
+ COUNTER_UPDATE(Parent->_scan_rows, stats.raw_rows_read);
\
+ _scan_rows += _tablet_reader->mutable_stats()->raw_rows_read;
\
COUNTER_UPDATE(Parent->_vec_cond_timer, stats.vec_cond_ns);
\
COUNTER_UPDATE(Parent->_short_cond_timer, stats.short_cond_ns);
\
COUNTER_UPDATE(Parent->_expr_filter_timer, stats.expr_filter_ns);
\
@@ -663,11 +665,11 @@ void NewOlapScanner::_collect_profile_before_close() {
#undef INCR_COUNTER
#endif
// Update metrics
-
DorisMetrics::instance()->query_scan_bytes->increment(_compressed_bytes_read);
- DorisMetrics::instance()->query_scan_rows->increment(_raw_rows_read);
+ DorisMetrics::instance()->query_scan_bytes->increment(_scan_bytes);
+ DorisMetrics::instance()->query_scan_rows->increment(_scan_rows);
auto& tablet = _tablet_reader_params.tablet;
- tablet->query_scan_bytes->increment(_compressed_bytes_read);
- tablet->query_scan_rows->increment(_raw_rows_read);
+ tablet->query_scan_bytes->increment(_scan_bytes);
+ tablet->query_scan_rows->increment(_scan_rows);
tablet->query_scan_count->increment(1);
if (_query_statistics) {
_query_statistics->add_scan_bytes_from_local_storage(
diff --git a/be/src/vec/exec/scan/new_olap_scanner.h
b/be/src/vec/exec/scan/new_olap_scanner.h
index 52c664c026a..44c300f446e 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.h
+++ b/be/src/vec/exec/scan/new_olap_scanner.h
@@ -101,8 +101,8 @@ private:
std::unordered_set<uint32_t> _tablet_columns_convert_to_null_set;
// ========= profiles ==========
- int64_t _compressed_bytes_read = 0;
- int64_t _raw_rows_read = 0;
+ int64_t _scan_bytes = 0;
+ int64_t _scan_rows = 0;
bool _profile_updated = false;
};
} // namespace vectorized
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
