This is an automated email from the ASF dual-hosted git repository.
panxiaolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new d9db3f5431 [Improvement](scan) Remove redundant predicates on scan
node (#23374)
d9db3f5431 is described below
commit d9db3f54312010ccb0d97a9db36273a83b5f8eef
Author: Pxl <pxl...@qq.com>
AuthorDate: Fri Aug 25 10:41:37 2023 +0800
[Improvement](scan) Remove redundant predicates on scan node (#23374)
* Remove redundant predicates on scan node
* update
* fix
---
be/src/pipeline/exec/olap_scan_operator.cpp | 7 +++++--
be/src/pipeline/exec/olap_scan_operator.h | 2 ++
be/src/pipeline/exec/scan_operator.cpp | 2 +-
be/src/pipeline/exec/scan_operator.h | 5 +++--
be/src/vec/exec/scan/new_olap_scan_node.cpp | 10 +---------
be/src/vec/exec/scan/new_olap_scan_node.h | 11 ++++++++++-
be/src/vec/exec/scan/vscan_node.cpp | 3 ++-
be/src/vec/exec/scan/vscan_node.h | 2 ++
8 files changed, 26 insertions(+), 16 deletions(-)
diff --git a/be/src/pipeline/exec/olap_scan_operator.cpp
b/be/src/pipeline/exec/olap_scan_operator.cpp
index 3156082ae6..5eca2f2b16 100644
--- a/be/src/pipeline/exec/olap_scan_operator.cpp
+++ b/be/src/pipeline/exec/olap_scan_operator.cpp
@@ -200,9 +200,12 @@ Status
OlapScanLocalState::_should_push_down_function_filter(
}
bool OlapScanLocalState::_should_push_down_common_expr() {
+ return state()->enable_common_expr_pushdown() && _storage_no_merge();
+}
+
+bool OlapScanLocalState::_storage_no_merge() {
auto& p = _parent->cast<OlapScanOperatorX>();
- return state()->enable_common_expr_pushdown() &&
- (p._olap_scan_node.keyType == TKeysType::DUP_KEYS ||
+ return (p._olap_scan_node.keyType == TKeysType::DUP_KEYS ||
(p._olap_scan_node.keyType == TKeysType::UNIQUE_KEYS &&
p._olap_scan_node.__isset.enable_unique_key_merge_on_write &&
p._olap_scan_node.enable_unique_key_merge_on_write));
diff --git a/be/src/pipeline/exec/olap_scan_operator.h
b/be/src/pipeline/exec/olap_scan_operator.h
index 7f11ba3a80..aec50d5d2f 100644
--- a/be/src/pipeline/exec/olap_scan_operator.h
+++ b/be/src/pipeline/exec/olap_scan_operator.h
@@ -72,6 +72,8 @@ private:
bool _should_push_down_common_expr() override;
+ bool _storage_no_merge() override;
+
Status _init_scanners(std::list<vectorized::VScannerSPtr>* scanners)
override;
void add_filter_info(int id, const PredicateFilterInfo& info);
diff --git a/be/src/pipeline/exec/scan_operator.cpp
b/be/src/pipeline/exec/scan_operator.cpp
index 1ecc9c0598..84f63c868f 100644
--- a/be/src/pipeline/exec/scan_operator.cpp
+++ b/be/src/pipeline/exec/scan_operator.cpp
@@ -356,7 +356,7 @@ Status ScanLocalState::_normalize_predicate(const
vectorized::VExprSPtr& conjunc
}
if (pdt == vectorized::VScanNode::PushDownType::ACCEPTABLE &&
- _is_key_column(slot->col_name())) {
+ (_is_key_column(slot->col_name()) || _storage_no_merge())) {
output_expr = nullptr;
return Status::OK();
} else {
diff --git a/be/src/pipeline/exec/scan_operator.h
b/be/src/pipeline/exec/scan_operator.h
index 8ad370b4a9..210a285605 100644
--- a/be/src/pipeline/exec/scan_operator.h
+++ b/be/src/pipeline/exec/scan_operator.h
@@ -59,7 +59,7 @@ class ScanLocalState : public PipelineXLocalState, public
vectorized::RuntimeFil
ENABLE_FACTORY_CREATOR(ScanLocalState);
ScanLocalState(RuntimeState* state, OperatorXBase* parent);
- virtual Status init(RuntimeState* state, LocalStateInfo& info) override;
+ Status init(RuntimeState* state, LocalStateInfo& info) override;
bool ready_to_read();
@@ -89,8 +89,9 @@ protected:
RETURN_IF_ERROR(_normalize_conjuncts());
return Status::OK();
}
-
virtual bool _should_push_down_common_expr() { return false; }
+
+ virtual bool _storage_no_merge() { return true; }
virtual bool _is_key_column(const std::string& col_name) { return false; }
virtual vectorized::VScanNode::PushDownType
_should_push_down_bloom_filter() {
return vectorized::VScanNode::PushDownType::UNACCEPTABLE;
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.cpp
b/be/src/vec/exec/scan/new_olap_scan_node.cpp
index 3af5bb9f89..31971e42c0 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.cpp
+++ b/be/src/vec/exec/scan/new_olap_scan_node.cpp
@@ -382,14 +382,6 @@ Status
NewOlapScanNode::_should_push_down_function_filter(VectorizedFnCall* fn_c
return Status::OK();
}
-bool NewOlapScanNode::_should_push_down_common_expr() {
- return _state->enable_common_expr_pushdown() &&
- (_olap_scan_node.keyType == TKeysType::DUP_KEYS ||
- (_olap_scan_node.keyType == TKeysType::UNIQUE_KEYS &&
- _olap_scan_node.__isset.enable_unique_key_merge_on_write &&
- _olap_scan_node.enable_unique_key_merge_on_write));
-}
-
// PlanFragmentExecutor will call this method to set scan range
// Doris scan range is defined in thrift file like this
// struct TPaloScanRange {
@@ -434,7 +426,7 @@ Status
NewOlapScanNode::_init_scanners(std::list<VScannerSPtr>* scanners) {
message += conjunct->root()->debug_string();
}
}
- _runtime_profile->add_info_string("RemainedDownPredicates", message);
+ _runtime_profile->add_info_string("RemainedPredicates", message);
}
if (!_olap_scan_node.output_column_unique_ids.empty()) {
diff --git a/be/src/vec/exec/scan/new_olap_scan_node.h
b/be/src/vec/exec/scan/new_olap_scan_node.h
index 8cfa08131d..0725c37cf5 100644
--- a/be/src/vec/exec/scan/new_olap_scan_node.h
+++ b/be/src/vec/exec/scan/new_olap_scan_node.h
@@ -85,7 +85,16 @@ protected:
PushDownType _should_push_down_is_null_predicate() override { return
PushDownType::ACCEPTABLE; }
- bool _should_push_down_common_expr() override;
+ bool _should_push_down_common_expr() override {
+ return _state->enable_common_expr_pushdown() && _storage_no_merge();
+ }
+
+ bool _storage_no_merge() override {
+ return (_olap_scan_node.keyType == TKeysType::DUP_KEYS ||
+ (_olap_scan_node.keyType == TKeysType::UNIQUE_KEYS &&
+ _olap_scan_node.__isset.enable_unique_key_merge_on_write &&
+ _olap_scan_node.enable_unique_key_merge_on_write));
+ }
Status _init_scanners(std::list<VScannerSPtr>* scanners) override;
diff --git a/be/src/vec/exec/scan/vscan_node.cpp
b/be/src/vec/exec/scan/vscan_node.cpp
index 580015d067..612dc32b22 100644
--- a/be/src/vec/exec/scan/vscan_node.cpp
+++ b/be/src/vec/exec/scan/vscan_node.cpp
@@ -549,7 +549,8 @@ Status VScanNode::_normalize_predicate(const VExprSPtr&
conjunct_expr_root, VExp
return Status::OK();
}
- if (pdt == PushDownType::ACCEPTABLE &&
_is_key_column(slot->col_name())) {
+ if (pdt == PushDownType::ACCEPTABLE &&
+ (_is_key_column(slot->col_name()) || _storage_no_merge())) {
output_expr = nullptr;
return Status::OK();
} else {
diff --git a/be/src/vec/exec/scan/vscan_node.h
b/be/src/vec/exec/scan/vscan_node.h
index ca95685c66..fcf8a7dd32 100644
--- a/be/src/vec/exec/scan/vscan_node.h
+++ b/be/src/vec/exec/scan/vscan_node.h
@@ -228,6 +228,8 @@ protected:
virtual bool _should_push_down_common_expr() { return false; }
+ virtual bool _storage_no_merge() { return false; }
+
virtual PushDownType _should_push_down_bloom_filter() { return
PushDownType::UNACCEPTABLE; }
virtual PushDownType _should_push_down_bitmap_filter() { return
PushDownType::UNACCEPTABLE; }
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
