This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 90c1e58141d [Chore](profile) add some profile on ReaderInit (#45556)
(#46964)
90c1e58141d is described below
commit 90c1e58141df83ad89e180b7cfcbce055b8a2ca4
Author: Pxl <x...@selectdb.com>
AuthorDate: Tue Jan 14 18:01:14 2025 +0800
[Chore](profile) add some profile on ReaderInit (#45556) (#46964)
pick from #45556
---
be/src/olap/base_tablet.cpp | 5 +-
be/src/olap/olap_common.h | 24 +++++++++
be/src/olap/rowset/beta_rowset_reader.cpp | 21 +++++---
be/src/olap/rowset/segment_v2/segment.cpp | 26 ++++++----
be/src/olap/rowset/segment_v2/segment.h | 6 ++-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 10 ++--
be/src/olap/tablet_reader.cpp | 7 +++
be/src/pipeline/exec/olap_scan_operator.cpp | 41 ++++++++++++++++
be/src/pipeline/exec/olap_scan_operator.h | 27 ++++++++++
be/src/vec/exec/scan/new_olap_scanner.cpp | 41 ++++++++++++++++
be/src/vec/olap/block_reader.cpp | 57 +++++++++++++---------
be/src/vec/olap/vgeneric_iterators.cpp | 5 +-
12 files changed, 221 insertions(+), 49 deletions(-)
diff --git a/be/src/olap/base_tablet.cpp b/be/src/olap/base_tablet.cpp
index 8cad5b87f5b..8dc6f3600a5 100644
--- a/be/src/olap/base_tablet.cpp
+++ b/be/src/olap/base_tablet.cpp
@@ -23,6 +23,7 @@
#include "common/status.h"
#include "olap/calc_delete_bitmap_executor.h"
#include "olap/delete_bitmap_calculator.h"
+#include "olap/iterators.h"
#include "olap/memtable.h"
#include "olap/partial_update_info.h"
#include "olap/primary_key_index.h"
@@ -74,7 +75,9 @@ Status _get_segment_column_iterator(const
BetaRowsetSharedPtr& rowset, uint32_t
rowset->rowset_id().to_string(),
segid));
}
segment_v2::SegmentSharedPtr segment = *it;
- RETURN_IF_ERROR(segment->new_column_iterator(target_column,
column_iterator, nullptr));
+ StorageReadOptions opts;
+ opts.stats = stats;
+ RETURN_IF_ERROR(segment->new_column_iterator(target_column,
column_iterator, &opts));
segment_v2::ColumnIteratorOptions opt {
.use_page_cache = !config::disable_storage_page_cache,
.file_reader = segment->file_reader().get(),
diff --git a/be/src/olap/olap_common.h b/be/src/olap/olap_common.h
index 7a62f7d87f7..b2e63a777ba 100644
--- a/be/src/olap/olap_common.h
+++ b/be/src/olap/olap_common.h
@@ -388,6 +388,30 @@ struct OlapReaderStatistics {
int64_t collect_iterator_merge_next_timer = 0;
int64_t collect_iterator_normal_next_timer = 0;
int64_t delete_bitmap_get_agg_ns = 0;
+
+ int64_t tablet_reader_init_timer_ns = 0;
+ int64_t tablet_reader_capture_rs_readers_timer_ns = 0;
+ int64_t tablet_reader_init_return_columns_timer_ns = 0;
+ int64_t tablet_reader_init_keys_param_timer_ns = 0;
+ int64_t tablet_reader_init_orderby_keys_param_timer_ns = 0;
+ int64_t tablet_reader_init_conditions_param_timer_ns = 0;
+ int64_t tablet_reader_init_delete_condition_param_timer_ns = 0;
+ int64_t block_reader_vcollect_iter_init_timer_ns = 0;
+ int64_t block_reader_rs_readers_init_timer_ns = 0;
+ int64_t block_reader_build_heap_init_timer_ns = 0;
+
+ int64_t rowset_reader_get_segment_iterators_timer_ns = 0;
+ int64_t rowset_reader_create_iterators_timer_ns = 0;
+ int64_t rowset_reader_init_iterators_timer_ns = 0;
+ int64_t rowset_reader_load_segments_timer_ns = 0;
+
+ int64_t segment_iterator_init_timer_ns = 0;
+ int64_t segment_iterator_init_return_column_iterators_timer_ns = 0;
+ int64_t segment_iterator_init_bitmap_index_iterators_timer_ns = 0;
+ int64_t segment_iterator_init_inverted_index_iterators_timer_ns = 0;
+
+ int64_t segment_create_column_readers_timer_ns = 0;
+ int64_t segment_load_index_timer_ns = 0;
};
using ColumnId = uint32_t;
diff --git a/be/src/olap/rowset/beta_rowset_reader.cpp
b/be/src/olap/rowset/beta_rowset_reader.cpp
index 042893f1374..d690b9b58d5 100644
--- a/be/src/olap/rowset/beta_rowset_reader.cpp
+++ b/be/src/olap/rowset/beta_rowset_reader.cpp
@@ -78,7 +78,6 @@ bool BetaRowsetReader::update_profile(RuntimeProfile*
profile) {
Status BetaRowsetReader::get_segment_iterators(RowsetReaderContext*
read_context,
std::vector<RowwiseIteratorUPtr>* out_iters,
bool use_cache) {
- RETURN_IF_ERROR(_rowset->load());
_read_context = read_context;
// The segment iterator is created with its own statistics,
// and the member variable '_stats' is initialized by
'_stats(&owned_stats)'.
@@ -92,6 +91,9 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
if (_read_context->stats != nullptr) {
_stats = _read_context->stats;
}
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_get_segment_iterators_timer_ns);
+
+ RETURN_IF_ERROR(_rowset->load());
// convert RowsetReaderContext to StorageReadOptions
_read_options.block_row_max = read_context->batch_size;
@@ -225,9 +227,12 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
bool should_use_cache = use_cache || (_read_context->reader_type ==
ReaderType::READER_QUERY &&
enable_segment_cache);
SegmentCacheHandle segment_cache_handle;
- RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(_rowset,
&segment_cache_handle,
- should_use_cache,
-
/*need_load_pk_index_and_bf*/ false));
+ {
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_load_segments_timer_ns);
+ RETURN_IF_ERROR(SegmentLoader::instance()->load_segments(
+ _rowset, &segment_cache_handle, should_use_cache,
+ /*need_load_pk_index_and_bf*/ false));
+ }
// create iterator for each segment
auto& segments = segment_cache_handle.get_segments();
@@ -252,6 +257,7 @@ Status
BetaRowsetReader::get_segment_iterators(RowsetReaderContext* read_context
const bool use_lazy_init_iterators =
!is_merge_iterator && _read_context->reader_type ==
ReaderType::READER_QUERY;
for (int i = seg_start; i < seg_end; i++) {
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_create_iterators_timer_ns);
auto& seg_ptr = segments[i];
std::unique_ptr<RowwiseIterator> iter;
@@ -316,6 +322,8 @@ Status BetaRowsetReader::_init_iterator() {
std::vector<RowwiseIteratorUPtr> iterators;
RETURN_IF_ERROR(get_segment_iterators(_read_context, &iterators));
+ SCOPED_RAW_TIMER(&_stats->rowset_reader_init_iterators_timer_ns);
+
if (_read_context->merged_rows == nullptr) {
_read_context->merged_rows = &_merged_rows;
}
@@ -351,8 +359,8 @@ Status BetaRowsetReader::_init_iterator() {
}
Status BetaRowsetReader::next_block(vectorized::Block* block) {
- SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RETURN_IF_ERROR(_init_iterator_once());
+ SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
if (_empty) {
return Status::Error<END_OF_FILE>("BetaRowsetReader is empty");
}
@@ -380,9 +388,8 @@ Status BetaRowsetReader::next_block(vectorized::Block*
block) {
}
Status BetaRowsetReader::next_block_view(vectorized::BlockView* block_view) {
- SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RETURN_IF_ERROR(_init_iterator_once());
-
+ SCOPED_RAW_TIMER(&_stats->block_fetch_ns);
RuntimeState* runtime_state = nullptr;
if (_read_context != nullptr) {
runtime_state = _read_context->runtime_state;
diff --git a/be/src/olap/rowset/segment_v2/segment.cpp
b/be/src/olap/rowset/segment_v2/segment.cpp
index f7c43ea4df4..2068e07a2a0 100644
--- a/be/src/olap/rowset/segment_v2/segment.cpp
+++ b/be/src/olap/rowset/segment_v2/segment.cpp
@@ -218,7 +218,7 @@ Status Segment::new_iterator(SchemaSPtr schema, const
StorageReadOptions& read_o
if (read_options.runtime_state != nullptr) {
_be_exec_version = read_options.runtime_state->be_exec_version();
}
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
read_options.stats->total_segment_number++;
// trying to prune the current segment by segment-level zone map
@@ -278,7 +278,11 @@ Status Segment::new_iterator(SchemaSPtr schema, const
StorageReadOptions& read_o
}
}
- RETURN_IF_ERROR(load_index(read_options.stats));
+ {
+ SCOPED_RAW_TIMER(&read_options.stats->segment_load_index_timer_ns);
+ RETURN_IF_ERROR(load_index(read_options.stats));
+ }
+
if (read_options.delete_condition_predicates->num_of_column_predicate() ==
0 &&
read_options.push_down_agg_type_opt != TPushAggOp::NONE &&
read_options.push_down_agg_type_opt != TPushAggOp::COUNT_ON_INDEX) {
@@ -584,7 +588,8 @@ vectorized::DataTypePtr Segment::get_data_type_of(const
ColumnIdentifier& identi
return nullptr;
}
-Status Segment::_create_column_readers_once() {
+Status Segment::_create_column_readers_once(OlapReaderStatistics* stats) {
+ SCOPED_RAW_TIMER(&stats->segment_create_column_readers_timer_ns);
return _create_column_readers_once_call.call([&] {
DCHECK(_footer_pb);
Defer defer([&]() { _footer_pb.reset(); });
@@ -858,10 +863,10 @@ Status Segment::new_column_iterator_with_path(const
TabletColumn& tablet_column,
Status Segment::new_column_iterator(const TabletColumn& tablet_column,
std::unique_ptr<ColumnIterator>* iter,
const StorageReadOptions* opt) {
- if (opt != nullptr && opt->runtime_state != nullptr) {
+ if (opt->runtime_state != nullptr) {
_be_exec_version = opt->runtime_state->be_exec_version();
}
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(opt->stats));
// init column iterator by path info
if (tablet_column.has_path_info() || tablet_column.is_variant_type()) {
@@ -889,8 +894,9 @@ Status Segment::new_column_iterator(const TabletColumn&
tablet_column,
return Status::OK();
}
-Status Segment::new_column_iterator(int32_t unique_id,
std::unique_ptr<ColumnIterator>* iter) {
- RETURN_IF_ERROR(_create_column_readers_once());
+Status Segment::new_column_iterator(int32_t unique_id, const
StorageReadOptions* opt,
+ std::unique_ptr<ColumnIterator>* iter) {
+ RETURN_IF_ERROR(_create_column_readers_once(opt->stats));
ColumnIterator* it;
RETURN_IF_ERROR(_column_readers.at(unique_id)->new_iterator(&it));
iter->reset(it);
@@ -918,8 +924,9 @@ ColumnReader* Segment::_get_column_reader(const
TabletColumn& col) {
}
Status Segment::new_bitmap_index_iterator(const TabletColumn& tablet_column,
+ const StorageReadOptions&
read_options,
std::unique_ptr<BitmapIndexIterator>* iter) {
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
ColumnReader* reader = _get_column_reader(tablet_column);
if (reader != nullptr && reader->has_bitmap_index()) {
BitmapIndexIterator* it;
@@ -937,7 +944,7 @@ Status Segment::new_inverted_index_iterator(const
TabletColumn& tablet_column,
if (read_options.runtime_state != nullptr) {
_be_exec_version = read_options.runtime_state->be_exec_version();
}
- RETURN_IF_ERROR(_create_column_readers_once());
+ RETURN_IF_ERROR(_create_column_readers_once(read_options.stats));
ColumnReader* reader = _get_column_reader(tablet_column);
if (reader != nullptr && index_meta) {
if (_inverted_index_file_reader == nullptr) {
@@ -1096,6 +1103,7 @@ Status Segment::seek_and_read_by_rowid(const
TabletSchema& schema, SlotDescripto
OlapReaderStatistics& stats,
std::unique_ptr<ColumnIterator>&
iterator_hint) {
StorageReadOptions storage_read_opt;
+ storage_read_opt.stats = &stats;
storage_read_opt.io_ctx.reader_type = ReaderType::READER_QUERY;
segment_v2::ColumnIteratorOptions opt {
.use_page_cache = !config::disable_storage_page_cache,
diff --git a/be/src/olap/rowset/segment_v2/segment.h
b/be/src/olap/rowset/segment_v2/segment.h
index 931ddd31c56..5ce8a49e3dc 100644
--- a/be/src/olap/rowset/segment_v2/segment.h
+++ b/be/src/olap/rowset/segment_v2/segment.h
@@ -111,9 +111,11 @@ public:
std::unique_ptr<ColumnIterator>* iter,
const StorageReadOptions* opt);
- Status new_column_iterator(int32_t unique_id,
std::unique_ptr<ColumnIterator>* iter);
+ Status new_column_iterator(int32_t unique_id, const StorageReadOptions*
opt,
+ std::unique_ptr<ColumnIterator>* iter);
Status new_bitmap_index_iterator(const TabletColumn& tablet_column,
+ const StorageReadOptions& read_options,
std::unique_ptr<BitmapIndexIterator>*
iter);
Status new_inverted_index_iterator(const TabletColumn& tablet_column,
@@ -234,7 +236,7 @@ private:
Status _open_inverted_index();
- Status _create_column_readers_once();
+ Status _create_column_readers_once(OlapReaderStatistics* stats);
private:
friend class SegmentIterator;
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 555f1b9893c..69d73e4b598 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -281,9 +281,10 @@ Status SegmentIterator::_init_impl(const
StorageReadOptions& opts) {
if (_inited) {
return Status::OK();
}
+ _opts = opts;
+ SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_timer_ns);
_inited = true;
_file_reader = _segment->_file_reader;
- _opts = opts;
_col_predicates.clear();
for (const auto& predicate : opts.column_predicates) {
@@ -1009,6 +1010,7 @@ bool
SegmentIterator::_check_all_conditions_passed_inverted_index_for_column(Col
}
Status SegmentIterator::_init_return_column_iterators() {
+
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_return_column_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
@@ -1051,19 +1053,21 @@ Status SegmentIterator::_init_return_column_iterators()
{
}
Status SegmentIterator::_init_bitmap_index_iterators() {
+
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_bitmap_index_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
for (auto cid : _schema->column_ids()) {
if (_bitmap_index_iterators[cid] == nullptr) {
-
RETURN_IF_ERROR(_segment->new_bitmap_index_iterator(_opts.tablet_schema->column(cid),
-
&_bitmap_index_iterators[cid]));
+ RETURN_IF_ERROR(_segment->new_bitmap_index_iterator(
+ _opts.tablet_schema->column(cid), _opts,
&_bitmap_index_iterators[cid]));
}
}
return Status::OK();
}
Status SegmentIterator::_init_inverted_index_iterators() {
+
SCOPED_RAW_TIMER(&_opts.stats->segment_iterator_init_inverted_index_iterators_timer_ns);
if (_cur_rowid >= num_rows()) {
return Status::OK();
}
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 7410b70f4aa..f4645ef202a 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -120,6 +120,7 @@ TabletReader::~TabletReader() {
}
Status TabletReader::init(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_timer_ns);
_predicate_arena = std::make_unique<vectorized::Arena>();
Status res = _init_params(read_params);
@@ -159,6 +160,7 @@ bool TabletReader::_optimize_for_single_rowset(
}
Status TabletReader::_capture_rs_readers(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_capture_rs_readers_timer_ns);
if (read_params.rs_splits.empty()) {
return Status::InternalError("fail to acquire data sources. tablet={}",
_tablet->tablet_id());
@@ -331,6 +333,7 @@ Status TabletReader::_init_params(const ReaderParams&
read_params) {
}
Status TabletReader::_init_return_columns(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_return_columns_timer_ns);
if (read_params.reader_type == ReaderType::READER_QUERY) {
_return_columns = read_params.return_columns;
_tablet_columns_convert_to_null_set =
read_params.tablet_columns_convert_to_null_set;
@@ -387,6 +390,7 @@ Status TabletReader::_init_return_columns(const
ReaderParams& read_params) {
}
Status TabletReader::_init_keys_param(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_keys_param_timer_ns);
if (read_params.start_key.empty()) {
return Status::OK();
}
@@ -461,6 +465,7 @@ Status TabletReader::_init_keys_param(const ReaderParams&
read_params) {
}
Status TabletReader::_init_orderby_keys_param(const ReaderParams& read_params)
{
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_orderby_keys_param_timer_ns);
// UNIQUE_KEYS will compare all keys as before
if (_tablet_schema->keys_type() == DUP_KEYS ||
(_tablet_schema->keys_type() == UNIQUE_KEYS &&
_tablet->enable_unique_key_merge_on_write())) {
@@ -487,6 +492,7 @@ Status TabletReader::_init_orderby_keys_param(const
ReaderParams& read_params) {
}
Status TabletReader::_init_conditions_param(const ReaderParams& read_params) {
+ SCOPED_RAW_TIMER(&_stats.tablet_reader_init_conditions_param_timer_ns);
std::vector<ColumnPredicate*> predicates;
for (const auto& condition : read_params.conditions) {
TCondition tmp_cond = condition;
@@ -617,6 +623,7 @@ ColumnPredicate* TabletReader::_parse_to_predicate(const
FunctionFilter& functio
}
Status TabletReader::_init_delete_condition(const ReaderParams& read_params) {
+
SCOPED_RAW_TIMER(&_stats.tablet_reader_init_delete_condition_param_timer_ns);
// If it's cumu and not allow do delete when cumu
if (read_params.reader_type == ReaderType::READER_SEGMENT_COMPACTION ||
(read_params.reader_type == ReaderType::READER_CUMULATIVE_COMPACTION &&
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index 05e8c27b8a2..137562e21d0 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -149,6 +149,47 @@ Status OlapScanLocalState::_init_profile() {
_tablet_counter = ADD_COUNTER(_runtime_profile, "TabletNum", TUnit::UNIT);
_key_range_counter = ADD_COUNTER(_runtime_profile, "KeyRangesNum",
TUnit::UNIT);
_runtime_filter_info = ADD_LABEL_COUNTER_WITH_LEVEL(_runtime_profile,
"RuntimeFilterInfo", 1);
+
+ _tablet_reader_init_timer = ADD_TIMER(_scanner_profile,
"TabletReaderInitTimer");
+ _tablet_reader_capture_rs_readers_timer =
+ ADD_TIMER(_scanner_profile, "TabletReaderCaptureRsReadersTimer");
+ _tablet_reader_init_return_columns_timer =
+ ADD_TIMER(_scanner_profile, "TabletReaderInitReturnColumnsTimer");
+ _tablet_reader_init_keys_param_timer =
+ ADD_TIMER(_scanner_profile, "TabletReaderInitKeysParamTimer");
+ _tablet_reader_init_orderby_keys_param_timer =
+ ADD_TIMER(_scanner_profile,
"TabletReaderInitOrderbyKeysParamTimer");
+ _tablet_reader_init_conditions_param_timer =
+ ADD_TIMER(_scanner_profile,
"TabletReaderInitConditionsParamTimer");
+ _tablet_reader_init_delete_condition_param_timer =
+ ADD_TIMER(_scanner_profile,
"TabletReaderInitDeleteConditionParamTimer");
+ _block_reader_vcollect_iter_init_timer =
+ ADD_TIMER(_scanner_profile, "BlockReaderVcollectIterInitTimer");
+ _block_reader_rs_readers_init_timer =
+ ADD_TIMER(_scanner_profile, "BlockReaderRsReadersInitTimer");
+ _block_reader_build_heap_init_timer =
+ ADD_TIMER(_scanner_profile, "BlockReaderBuildHeapInitTimer");
+
+ _rowset_reader_get_segment_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"RowsetReaderGetSegmentIteratorsTimer");
+ _rowset_reader_create_iterators_timer =
+ ADD_TIMER(_scanner_profile, "RowsetReaderCreateIteratorsTimer");
+ _rowset_reader_init_iterators_timer =
+ ADD_TIMER(_scanner_profile, "RowsetReaderInitIteratorsTimer");
+ _rowset_reader_load_segments_timer =
+ ADD_TIMER(_scanner_profile, "RowsetReaderLoadSegmentsTimer");
+
+ _segment_iterator_init_timer = ADD_TIMER(_scanner_profile,
"SegmentIteratorInitTimer");
+ _segment_iterator_init_return_column_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"SegmentIteratorInitReturnColumnIteratorsTimer");
+ _segment_iterator_init_bitmap_index_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"SegmentIteratorInitBitmapIndexIteratorsTimer");
+ _segment_iterator_init_inverted_index_iterators_timer =
+ ADD_TIMER(_scanner_profile,
"SegmentIteratorInitInvertedIndexIteratorsTimer");
+
+ _segment_create_column_readers_timer =
+ ADD_TIMER(_scanner_profile, "SegmentCreateColumnReadersTimer");
+ _segment_load_index_timer = ADD_TIMER(_scanner_profile,
"SegmentLoadIndexTimer");
return Status::OK();
}
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index 69b4f3701b3..8559bfec2a8 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -183,6 +183,33 @@ private:
RuntimeProfile::Counter* _runtime_filter_info = nullptr;
+ // timer about tablet reader
+ RuntimeProfile::Counter* _tablet_reader_init_timer = nullptr;
+ RuntimeProfile::Counter* _tablet_reader_capture_rs_readers_timer = nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_return_columns_timer =
nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_keys_param_timer = nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_orderby_keys_param_timer =
nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_conditions_param_timer =
nullptr;
+ RuntimeProfile::Counter* _tablet_reader_init_delete_condition_param_timer
= nullptr;
+
+ // timer about block reader
+ RuntimeProfile::Counter* _block_reader_vcollect_iter_init_timer = nullptr;
+ RuntimeProfile::Counter* _block_reader_rs_readers_init_timer = nullptr;
+ RuntimeProfile::Counter* _block_reader_build_heap_init_timer = nullptr;
+
+ RuntimeProfile::Counter* _rowset_reader_get_segment_iterators_timer =
nullptr;
+ RuntimeProfile::Counter* _rowset_reader_create_iterators_timer = nullptr;
+ RuntimeProfile::Counter* _rowset_reader_init_iterators_timer = nullptr;
+ RuntimeProfile::Counter* _rowset_reader_load_segments_timer = nullptr;
+
+ RuntimeProfile::Counter* _segment_iterator_init_timer = nullptr;
+ RuntimeProfile::Counter*
_segment_iterator_init_return_column_iterators_timer = nullptr;
+ RuntimeProfile::Counter*
_segment_iterator_init_bitmap_index_iterators_timer = nullptr;
+ RuntimeProfile::Counter*
_segment_iterator_init_inverted_index_iterators_timer = nullptr;
+
+ RuntimeProfile::Counter* _segment_create_column_readers_timer = nullptr;
+ RuntimeProfile::Counter* _segment_load_index_timer = nullptr;
+
std::mutex _profile_mtx;
};
diff --git a/be/src/vec/exec/scan/new_olap_scanner.cpp
b/be/src/vec/exec/scan/new_olap_scanner.cpp
index f2ca2f0731b..09103da9481 100644
--- a/be/src/vec/exec/scan/new_olap_scanner.cpp
+++ b/be/src/vec/exec/scan/new_olap_scanner.cpp
@@ -648,6 +648,47 @@ void NewOlapScanner::_collect_profile_before_close() {
#undef INCR_COUNTER
#endif
+ COUNTER_UPDATE(local_state->_tablet_reader_init_timer,
stats.tablet_reader_init_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_capture_rs_readers_timer,
+ stats.tablet_reader_capture_rs_readers_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_init_return_columns_timer,
+ stats.tablet_reader_init_return_columns_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_init_keys_param_timer,
+ stats.tablet_reader_init_keys_param_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_init_orderby_keys_param_timer,
+ stats.tablet_reader_init_orderby_keys_param_timer_ns);
+ COUNTER_UPDATE(local_state->_tablet_reader_init_conditions_param_timer,
+ stats.tablet_reader_init_conditions_param_timer_ns);
+
COUNTER_UPDATE(local_state->_tablet_reader_init_delete_condition_param_timer,
+ stats.tablet_reader_init_delete_condition_param_timer_ns);
+ COUNTER_UPDATE(local_state->_block_reader_vcollect_iter_init_timer,
+ stats.block_reader_vcollect_iter_init_timer_ns);
+ COUNTER_UPDATE(local_state->_block_reader_rs_readers_init_timer,
+ stats.block_reader_rs_readers_init_timer_ns);
+ COUNTER_UPDATE(local_state->_block_reader_build_heap_init_timer,
+ stats.block_reader_build_heap_init_timer_ns);
+
+ COUNTER_UPDATE(local_state->_rowset_reader_get_segment_iterators_timer,
+ stats.rowset_reader_get_segment_iterators_timer_ns);
+ COUNTER_UPDATE(local_state->_rowset_reader_create_iterators_timer,
+ stats.rowset_reader_create_iterators_timer_ns);
+ COUNTER_UPDATE(local_state->_rowset_reader_init_iterators_timer,
+ stats.rowset_reader_init_iterators_timer_ns);
+ COUNTER_UPDATE(local_state->_rowset_reader_load_segments_timer,
+ stats.rowset_reader_load_segments_timer_ns);
+
+ COUNTER_UPDATE(local_state->_segment_iterator_init_timer,
stats.segment_iterator_init_timer_ns);
+
COUNTER_UPDATE(local_state->_segment_iterator_init_return_column_iterators_timer,
+
stats.segment_iterator_init_return_column_iterators_timer_ns);
+
COUNTER_UPDATE(local_state->_segment_iterator_init_bitmap_index_iterators_timer,
+
stats.segment_iterator_init_bitmap_index_iterators_timer_ns);
+
COUNTER_UPDATE(local_state->_segment_iterator_init_inverted_index_iterators_timer,
+
stats.segment_iterator_init_inverted_index_iterators_timer_ns);
+
+ COUNTER_UPDATE(local_state->_segment_create_column_readers_timer,
+ stats.segment_create_column_readers_timer_ns);
+ COUNTER_UPDATE(local_state->_segment_load_index_timer,
stats.segment_load_index_timer_ns);
+
// Update metrics
DorisMetrics::instance()->query_scan_bytes->increment(
local_state->_read_compressed_counter->value());
diff --git a/be/src/vec/olap/block_reader.cpp b/be/src/vec/olap/block_reader.cpp
index c46ff330f2b..07befd47d88 100644
--- a/be/src/vec/olap/block_reader.cpp
+++ b/be/src/vec/olap/block_reader.cpp
@@ -108,40 +108,49 @@ Status BlockReader::_init_collect_iter(const
ReaderParams& read_params) {
return res;
}
// check if rowsets are noneoverlapping
- _is_rowsets_overlapping = _rowsets_mono_asc_disjoint(read_params);
- _vcollect_iter.init(this, _is_rowsets_overlapping,
read_params.read_orderby_key,
- read_params.read_orderby_key_reverse);
+ {
+ SCOPED_RAW_TIMER(&_stats.block_reader_vcollect_iter_init_timer_ns);
+ _is_rowsets_overlapping = _rowsets_mono_asc_disjoint(read_params);
+ _vcollect_iter.init(this, _is_rowsets_overlapping,
read_params.read_orderby_key,
+ read_params.read_orderby_key_reverse);
+ }
std::vector<RowsetReaderSharedPtr> valid_rs_readers;
RuntimeState* runtime_state = read_params.runtime_state;
- for (int i = 0; i < read_params.rs_splits.size(); ++i) {
- if (runtime_state != nullptr && runtime_state->is_cancelled()) {
- return runtime_state->cancel_reason();
- }
+ {
+ SCOPED_RAW_TIMER(&_stats.block_reader_rs_readers_init_timer_ns);
+ for (int i = 0; i < read_params.rs_splits.size(); ++i) {
+ if (runtime_state != nullptr && runtime_state->is_cancelled()) {
+ return runtime_state->cancel_reason();
+ }
- auto& rs_split = read_params.rs_splits[i];
+ auto& rs_split = read_params.rs_splits[i];
- // _vcollect_iter.topn_next() will init rs_reader by itself
- if (!_vcollect_iter.use_topn_next()) {
- RETURN_IF_ERROR(rs_split.rs_reader->init(&_reader_context,
rs_split));
- }
+ // _vcollect_iter.topn_next() will init rs_reader by itself
+ if (!_vcollect_iter.use_topn_next()) {
+ RETURN_IF_ERROR(rs_split.rs_reader->init(&_reader_context,
rs_split));
+ }
- Status res = _vcollect_iter.add_child(rs_split);
- if (!res.ok() && !res.is<END_OF_FILE>()) {
- LOG(WARNING) << "failed to add child to iterator, err=" << res;
- return res;
- }
- if (res.ok()) {
- valid_rs_readers.push_back(rs_split.rs_reader);
+ Status res = _vcollect_iter.add_child(rs_split);
+ if (!res.ok() && !res.is<END_OF_FILE>()) {
+ LOG(WARNING) << "failed to add child to iterator, err=" << res;
+ return res;
+ }
+ if (res.ok()) {
+ valid_rs_readers.push_back(rs_split.rs_reader);
+ }
}
}
- RETURN_IF_ERROR(_vcollect_iter.build_heap(valid_rs_readers));
- // _vcollect_iter.topn_next() can not use current_row
- if (!_vcollect_iter.use_topn_next()) {
- auto status = _vcollect_iter.current_row(&_next_row);
- _eof = status.is<END_OF_FILE>();
+ {
+ SCOPED_RAW_TIMER(&_stats.block_reader_build_heap_init_timer_ns);
+ RETURN_IF_ERROR(_vcollect_iter.build_heap(valid_rs_readers));
+ // _vcollect_iter.topn_next() can not use current_row
+ if (!_vcollect_iter.use_topn_next()) {
+ auto status = _vcollect_iter.current_row(&_next_row);
+ _eof = status.is<END_OF_FILE>();
+ }
}
return Status::OK();
diff --git a/be/src/vec/olap/vgeneric_iterators.cpp
b/be/src/vec/olap/vgeneric_iterators.cpp
index d8a073fc11a..fe37abd08fa 100644
--- a/be/src/vec/olap/vgeneric_iterators.cpp
+++ b/be/src/vec/olap/vgeneric_iterators.cpp
@@ -50,9 +50,8 @@ Status VStatisticsIterator::init(const StorageReadOptions&
opts) {
auto cid = _schema.column_id(i);
auto unique_id = _schema.column(cid)->unique_id();
if (_column_iterators_map.count(unique_id) < 1) {
-
RETURN_IF_ERROR(_segment->new_column_iterator(opts.tablet_schema->column(cid),
-
&_column_iterators_map[unique_id],
- nullptr));
+ RETURN_IF_ERROR(_segment->new_column_iterator(
+ opts.tablet_schema->column(cid),
&_column_iterators_map[unique_id], &opts));
}
_column_iterators.push_back(_column_iterators_map[unique_id].get());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
