This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 245be9f33cf branch-4.0: [Chore](profile) attach always true counter to
column predicate #58463 (#58597)
245be9f33cf is described below
commit 245be9f33cf7206dcf1916411ed7aae6440d904f
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Dec 2 16:11:41 2025 +0800
branch-4.0: [Chore](profile) attach always true counter to column predicate
#58463 (#58597)
Cherry-picked from #58463
Co-authored-by: Pxl <[email protected]>
---
be/src/exec/olap_common.h | 29 ++++++++++++++--------
be/src/olap/bitmap_filter_predicate.h | 2 +-
be/src/olap/column_predicate.h | 21 +++++++++++++---
be/src/olap/comparison_predicate.h | 4 +--
be/src/olap/filter_olap_param.h | 8 +++++-
be/src/olap/rowset/segment_v2/segment_iterator.cpp | 8 ++----
be/src/olap/tablet_reader.cpp | 5 ++--
be/src/pipeline/exec/scan_operator.cpp | 16 +++++++++---
be/src/vec/exprs/vruntimefilter_wrapper.h | 3 +++
9 files changed, 66 insertions(+), 30 deletions(-)
diff --git a/be/src/exec/olap_common.h b/be/src/exec/olap_common.h
index 5d9f3741c51..8cb0311b326 100644
--- a/be/src/exec/olap_common.h
+++ b/be/src/exec/olap_common.h
@@ -210,7 +210,8 @@ public:
if (null_pred.condition_values.size() != 0) {
filters.emplace_back(_column_name, null_pred,
_runtime_filter_id,
_predicate_filtered_rows_counter,
- _predicate_input_rows_counter);
+ _predicate_input_rows_counter,
+ _predicate_always_true_rows_counter);
return;
}
@@ -223,9 +224,9 @@ public:
}
if (low.condition_values.size() != 0) {
- filters.emplace_back(_column_name, low, _runtime_filter_id,
- _predicate_filtered_rows_counter,
- _predicate_input_rows_counter);
+ filters.emplace_back(
+ _column_name, low, _runtime_filter_id,
_predicate_filtered_rows_counter,
+ _predicate_input_rows_counter,
_predicate_always_true_rows_counter);
}
TCondition high;
@@ -237,9 +238,9 @@ public:
}
if (high.condition_values.size() != 0) {
- filters.emplace_back(_column_name, high, _runtime_filter_id,
- _predicate_filtered_rows_counter,
- _predicate_input_rows_counter);
+ filters.emplace_back(
+ _column_name, high, _runtime_filter_id,
_predicate_filtered_rows_counter,
+ _predicate_input_rows_counter,
_predicate_always_true_rows_counter);
}
} else {
// 3. convert to is null and is not null filter condition
@@ -254,7 +255,8 @@ public:
if (null_pred.condition_values.size() != 0) {
filters.emplace_back(_column_name, null_pred,
_runtime_filter_id,
_predicate_filtered_rows_counter,
- _predicate_input_rows_counter);
+ _predicate_input_rows_counter,
+ _predicate_always_true_rows_counter);
}
}
}
@@ -271,7 +273,8 @@ public:
if (condition.condition_values.size() != 0) {
filters.emplace_back(_column_name, condition, _runtime_filter_id,
- _predicate_filtered_rows_counter,
_predicate_input_rows_counter);
+ _predicate_filtered_rows_counter,
_predicate_input_rows_counter,
+ _predicate_always_true_rows_counter);
}
}
@@ -311,7 +314,8 @@ public:
void attach_profile_counter(
int runtime_filter_id,
std::shared_ptr<RuntimeProfile::Counter>
predicate_filtered_rows_counter,
- std::shared_ptr<RuntimeProfile::Counter>
predicate_input_rows_counter) {
+ std::shared_ptr<RuntimeProfile::Counter>
predicate_input_rows_counter,
+ std::shared_ptr<RuntimeProfile::Counter>
predicate_always_true_rows_counter) {
DCHECK(predicate_filtered_rows_counter != nullptr);
DCHECK(predicate_input_rows_counter != nullptr);
@@ -323,6 +327,9 @@ public:
if (predicate_input_rows_counter != nullptr) {
_predicate_input_rows_counter = predicate_input_rows_counter;
}
+ if (predicate_always_true_rows_counter != nullptr) {
+ _predicate_always_true_rows_counter =
predicate_always_true_rows_counter;
+ }
}
int precision() const { return _precision; }
@@ -397,6 +404,8 @@ private:
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
std::shared_ptr<RuntimeProfile::Counter> _predicate_input_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
+ std::shared_ptr<RuntimeProfile::Counter>
_predicate_always_true_rows_counter =
+ std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
};
template <>
const typename ColumnValueRange<TYPE_FLOAT>::CppType
ColumnValueRange<TYPE_FLOAT>::TYPE_MIN;
diff --git a/be/src/olap/bitmap_filter_predicate.h
b/be/src/olap/bitmap_filter_predicate.h
index 9bf44354a98..48b26c69359 100644
--- a/be/src/olap/bitmap_filter_predicate.h
+++ b/be/src/olap/bitmap_filter_predicate.h
@@ -111,7 +111,7 @@ uint16_t
BitmapFilterColumnPredicate<T>::_evaluate_inner(const vectorized::IColu
} else {
new_size = evaluate<false>(column, nullptr, sel, size);
}
- update_filter_info(size - new_size, size);
+ update_filter_info(size - new_size, size, 0);
return new_size;
}
} //namespace doris
diff --git a/be/src/olap/column_predicate.h b/be/src/olap/column_predicate.h
index 6923ff435e3..daee6f10a3e 100644
--- a/be/src/olap/column_predicate.h
+++ b/be/src/olap/column_predicate.h
@@ -189,6 +189,7 @@ public:
Defer defer([&] { try_reset_judge_selectivity(); });
if (always_true()) {
+ update_filter_info(0, 0, size);
return size;
}
@@ -196,7 +197,7 @@ public:
if (_can_ignore()) {
do_judge_selectivity(size - new_size, size);
}
- update_filter_info(size - new_size, size);
+ update_filter_info(size - new_size, size, 0);
return new_size;
}
virtual void evaluate_and(const vectorized::IColumn& column, const
uint16_t* sel, uint16_t size,
@@ -259,7 +260,8 @@ public:
void attach_profile_counter(
int filter_id, std::shared_ptr<RuntimeProfile::Counter>
predicate_filtered_rows_counter,
- std::shared_ptr<RuntimeProfile::Counter>
predicate_input_rows_counter) {
+ std::shared_ptr<RuntimeProfile::Counter>
predicate_input_rows_counter,
+ std::shared_ptr<RuntimeProfile::Counter>
predicate_always_true_rows_counter) {
_runtime_filter_id = filter_id;
DCHECK(predicate_filtered_rows_counter != nullptr);
DCHECK(predicate_input_rows_counter != nullptr);
@@ -270,12 +272,22 @@ public:
if (predicate_input_rows_counter != nullptr) {
_predicate_input_rows_counter = predicate_input_rows_counter;
}
+ if (predicate_always_true_rows_counter != nullptr) {
+ _predicate_always_true_rows_counter =
predicate_always_true_rows_counter;
+ }
}
/// TODO: Currently we only record statistics for runtime filters, in the
future we should record for all predicates
- void update_filter_info(int64_t filter_rows, int64_t input_rows) const {
+ void update_filter_info(int64_t filter_rows, int64_t input_rows,
+ int64_t always_true_rows) const {
+ if (_predicate_input_rows_counter == nullptr ||
+ _predicate_filtered_rows_counter == nullptr ||
+ _predicate_always_true_rows_counter == nullptr) {
+ throw Exception(INTERNAL_ERROR, "Predicate profile counters are
not initialized");
+ }
COUNTER_UPDATE(_predicate_input_rows_counter, input_rows);
COUNTER_UPDATE(_predicate_filtered_rows_counter, filter_rows);
+ COUNTER_UPDATE(_predicate_always_true_rows_counter, always_true_rows);
}
static std::string pred_type_string(PredicateType type) {
@@ -367,9 +379,10 @@ protected:
std::shared_ptr<RuntimeProfile::Counter> _predicate_filtered_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
-
std::shared_ptr<RuntimeProfile::Counter> _predicate_input_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
+ std::shared_ptr<RuntimeProfile::Counter>
_predicate_always_true_rows_counter =
+ std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
};
} //namespace doris
diff --git a/be/src/olap/comparison_predicate.h
b/be/src/olap/comparison_predicate.h
index 7df41bf11fe..ba5be11de60 100644
--- a/be/src/olap/comparison_predicate.h
+++ b/be/src/olap/comparison_predicate.h
@@ -284,8 +284,8 @@ public:
// so reference here is safe.
//
https://stackoverflow.com/questions/14688285/c-local-variable-destruction-order
Defer defer([&]() {
- update_filter_info(current_evaluated_rows - current_passed_rows,
- current_evaluated_rows);
+ update_filter_info(current_evaluated_rows - current_passed_rows,
current_evaluated_rows,
+ 0);
try_reset_judge_selectivity();
});
diff --git a/be/src/olap/filter_olap_param.h b/be/src/olap/filter_olap_param.h
index 272fee63fb5..0818c088523 100644
--- a/be/src/olap/filter_olap_param.h
+++ b/be/src/olap/filter_olap_param.h
@@ -25,7 +25,8 @@ template <typename T>
struct FilterOlapParam {
FilterOlapParam(std::string column_name, T filter, int runtime_filter_id,
std::shared_ptr<RuntimeProfile::Counter> filtered_counter,
- std::shared_ptr<RuntimeProfile::Counter> input_counter)
+ std::shared_ptr<RuntimeProfile::Counter> input_counter,
+ std::shared_ptr<RuntimeProfile::Counter>
always_true_counter)
: column_name(std::move(column_name)),
filter(std::move(filter)),
runtime_filter_id(runtime_filter_id) {
@@ -37,6 +38,9 @@ struct FilterOlapParam {
if (input_counter != nullptr) {
input_rows_counter = input_counter;
}
+ if (always_true_counter != nullptr) {
+ always_true_rows_counter = always_true_counter;
+ }
}
std::string column_name;
@@ -46,6 +50,8 @@ struct FilterOlapParam {
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
std::shared_ptr<RuntimeProfile::Counter> input_rows_counter =
std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
+ std::shared_ptr<RuntimeProfile::Counter> always_true_rows_counter =
+ std::make_shared<RuntimeProfile::Counter>(TUnit::UNIT, 0);
};
} // namespace doris
diff --git a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
index 69e632b856c..d0301c7677f 100644
--- a/be/src/olap/rowset/segment_v2/segment_iterator.cpp
+++ b/be/src/olap/rowset/segment_v2/segment_iterator.cpp
@@ -2191,12 +2191,8 @@ uint16_t
SegmentIterator::_evaluate_vectorization_predicate(uint16_t* sel_rowid_
for (const auto& pred : _pre_eval_block_predicate) {
if (!pred->always_true()) {
all_pred_always_true = false;
- break;
- }
- }
- if (all_pred_always_true) {
- for (const auto& pred : _pre_eval_block_predicate) {
- DCHECK(pred->always_true());
+ } else {
+ pred->update_filter_info(0, 0, selected_size);
}
}
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 27668d93fef..00eec28cf8a 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -523,7 +523,8 @@ Status TabletReader::_init_conditions_param(const
ReaderParams& read_params) {
for (const auto& param : params) {
ColumnPredicate* predicate =
_parse_to_predicate({param.column_name, param.filter});
predicate->attach_profile_counter(param.runtime_filter_id,
param.filtered_rows_counter,
- param.input_rows_counter);
+ param.input_rows_counter,
+ param.always_true_rows_counter);
predicates.emplace_back(predicate);
}
};
@@ -540,7 +541,7 @@ Status TabletReader::_init_conditions_param(const
ReaderParams& read_params) {
// record condition value into predicate_params in order to pushdown
segment_iterator,
// _gen_predicate_result_sign will build predicate result unique sign
with condition value
predicate->attach_profile_counter(param.runtime_filter_id,
param.filtered_rows_counter,
- param.input_rows_counter);
+ param.input_rows_counter,
param.always_true_rows_counter);
predicates.emplace_back(predicate);
}
parse_and_emplace_predicates(read_params.bloom_filters);
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index 61e42c81503..05f127ba9df 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -321,7 +321,8 @@ Status ScanLocalState<Derived>::_normalize_predicate(
value_range.attach_profile_counter(
rf_expr->filter_id(),
rf_expr->predicate_filtered_rows_counter(),
-
rf_expr->predicate_input_rows_counter());
+
rf_expr->predicate_input_rows_counter(),
+
rf_expr->predicate_always_true_rows_counter());
}
}};
RETURN_IF_PUSH_DOWN(_normalize_in_and_eq_predicate(
@@ -388,7 +389,8 @@ Status
ScanLocalState<Derived>::_normalize_bloom_filter(vectorized::VExpr* expr,
_filter_predicates.bloom_filters.emplace_back(
slot->col_name(), expr->get_bloom_filter_func(),
rf_expr->filter_id(),
rf_expr->predicate_filtered_rows_counter(),
- rf_expr->predicate_input_rows_counter());
+ rf_expr->predicate_input_rows_counter(),
+ rf_expr->predicate_always_true_rows_counter());
*pdt = temp_pdt;
}
}
@@ -408,7 +410,8 @@ Status
ScanLocalState<Derived>::_normalize_bitmap_filter(vectorized::VExpr* expr
_filter_predicates.bitmap_filters.emplace_back(
slot->col_name(), expr->get_bitmap_filter_func(),
rf_expr->filter_id(),
rf_expr->predicate_filtered_rows_counter(),
- rf_expr->predicate_input_rows_counter());
+ rf_expr->predicate_input_rows_counter(),
+ rf_expr->predicate_always_true_rows_counter());
*pdt = temp_pdt;
}
}
@@ -620,16 +623,21 @@ Status
ScanLocalState<Derived>::_normalize_in_and_eq_predicate(vectorized::VExpr
int runtime_filter_id = -1;
std::shared_ptr<RuntimeProfile::Counter>
predicate_filtered_rows_counter = nullptr;
std::shared_ptr<RuntimeProfile::Counter>
predicate_input_rows_counter = nullptr;
+ std::shared_ptr<RuntimeProfile::Counter>
predicate_always_true_rows_counter =
+ nullptr;
if (expr_ctx->root()->is_rf_wrapper()) {
auto* rf_expr =
assert_cast<vectorized::VRuntimeFilterWrapper*>(expr_ctx->root().get());
runtime_filter_id = rf_expr->filter_id();
predicate_filtered_rows_counter =
rf_expr->predicate_filtered_rows_counter();
predicate_input_rows_counter =
rf_expr->predicate_input_rows_counter();
+ predicate_always_true_rows_counter =
+ rf_expr->predicate_always_true_rows_counter();
}
_filter_predicates.in_filters.emplace_back(
slot->col_name(), expr->get_set_func(),
runtime_filter_id,
- predicate_filtered_rows_counter,
predicate_input_rows_counter);
+ predicate_filtered_rows_counter,
predicate_input_rows_counter,
+ predicate_always_true_rows_counter);
*pdt = PushDownType::ACCEPTABLE;
return Status::OK();
}
diff --git a/be/src/vec/exprs/vruntimefilter_wrapper.h
b/be/src/vec/exprs/vruntimefilter_wrapper.h
index cdf30cff814..c68b773e883 100644
--- a/be/src/vec/exprs/vruntimefilter_wrapper.h
+++ b/be/src/vec/exprs/vruntimefilter_wrapper.h
@@ -116,6 +116,9 @@ public:
std::shared_ptr<RuntimeProfile::Counter> predicate_input_rows_counter()
const {
return _rf_input_rows;
}
+ std::shared_ptr<RuntimeProfile::Counter>
predicate_always_true_rows_counter() const {
+ return _always_true_filter_rows;
+ }
private:
void reset_judge_selectivity() const {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
