This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit f25af15842f9da0df6a1d66e9eecc9188080120e
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Fri Jan 26 17:41:48 2024 +0800
[Fix](Nereids) Fix lost predicate when query has a filter at the right
input of the outer join (#30374)
materialized view def is as following:
> select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey
> from lineitem
> left join (select * from orders where o_orderdate = '2023-12-10' )
t2
> on lineitem.l_orderkey = t2.o_orderkey;
the query as following, should add filter `o_orderdate = '2023-12-10'` on
mv when query rewrite by materialized view
> select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey
> from lineitem
> left join orders
> on lineitem.l_orderkey = orders.o_orderkey
> where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5;
---
.../mv/AbstractMaterializedViewRule.java | 66 ++++++++++++++--------
.../mv/join/left_outer/outer_join.out | 8 +++
.../mv/join/left_outer/outer_join.groovy | 24 +++++++-
3 files changed, 73 insertions(+), 25 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
index 57e06d4e16e..a6b6bdeeeb3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -441,6 +441,25 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
viewStructInfo = viewStructInfo.withPredicates(
viewStructInfo.getPredicates().merge(viewPulledUpExpressions));
}
+ // if the join type in query and mv plan is different, we should check
query is have the
+ // filters which rejects null
+ Set<Set<Slot>> requireNoNullableViewSlot =
comparisonResult.getViewNoNullableSlot();
+ // check query is use the null reject slot which view comparison need
+ if (!requireNoNullableViewSlot.isEmpty()) {
+ SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
+ // try to use
+ boolean valid = containsNullRejectSlot(requireNoNullableViewSlot,
+ queryStructInfo.getPredicates().getPulledUpPredicates(),
queryToViewMapping, cascadesContext);
+ if (!valid) {
+ queryStructInfo = queryStructInfo.withPredicates(
+
queryStructInfo.getPredicates().merge(comparisonResult.getQueryAllPulledUpExpressions()));
+ valid = containsNullRejectSlot(requireNoNullableViewSlot,
+
queryStructInfo.getPredicates().getPulledUpPredicates(), queryToViewMapping,
cascadesContext);
+ }
+ if (!valid) {
+ return SplitPredicate.INVALID_INSTANCE;
+ }
+ }
// viewEquivalenceClass to query based
// equal predicate compensate
final Set<Expression> equalCompensateConjunctions =
Predicates.compensateEquivalence(
@@ -464,30 +483,6 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
|| residualCompensatePredicates == null) {
return SplitPredicate.INVALID_INSTANCE;
}
- // if the join type in query and mv plan is different, we should check
query is have the
- // filters which rejects null
- Set<Set<Slot>> requireNoNullableViewSlot =
comparisonResult.getViewNoNullableSlot();
- // check query is use the null reject slot which view comparison need
- if (!requireNoNullableViewSlot.isEmpty()) {
- Set<Expression> queryPulledUpPredicates =
comparisonResult.getQueryAllPulledUpExpressions().stream()
- .flatMap(expr ->
ExpressionUtils.extractConjunction(expr).stream())
- .collect(Collectors.toSet());
- Set<Expression> nullRejectPredicates =
ExpressionUtils.inferNotNull(queryPulledUpPredicates,
- cascadesContext);
- SlotMapping queryToViewMapping = viewToQuerySlotMapping.inverse();
- Set<Expression> queryUsedNeedRejectNullSlotsViewBased =
nullRejectPredicates.stream()
- .map(expression ->
TypeUtils.isNotNull(expression).orElse(null))
- .filter(Objects::nonNull)
- .map(expr -> ExpressionUtils.replace((Expression) expr,
queryToViewMapping.toSlotReferenceMap()))
- .collect(Collectors.toSet());
- // query pulledUp predicates should have null reject predicates
and contains any require noNullable slot
- boolean valid =
!queryPulledUpPredicates.containsAll(nullRejectPredicates)
- && requireNoNullableViewSlot.stream().noneMatch(
- set -> Sets.intersection(set,
queryUsedNeedRejectNullSlotsViewBased).isEmpty());
- if (!valid) {
- return SplitPredicate.INVALID_INSTANCE;
- }
- }
return SplitPredicate.of(equalCompensateConjunctions.isEmpty() ?
BooleanLiteral.TRUE
: ExpressionUtils.and(equalCompensateConjunctions),
rangeCompensatePredicates.isEmpty() ? BooleanLiteral.TRUE
@@ -496,6 +491,29 @@ public abstract class AbstractMaterializedViewRule
implements ExplorationRuleFac
: ExpressionUtils.and(residualCompensatePredicates));
}
+ /**
+ * Check the queryPredicates contains the required nullable slot
+ */
+ private boolean containsNullRejectSlot(Set<Set<Slot>>
requireNoNullableViewSlot,
+ Set<Expression> queryPredicates,
+ SlotMapping queryToViewMapping,
+ CascadesContext cascadesContext) {
+ Set<Expression> queryPulledUpPredicates = queryPredicates.stream()
+ .flatMap(expr ->
ExpressionUtils.extractConjunction(expr).stream())
+ .collect(Collectors.toSet());
+ Set<Expression> nullRejectPredicates =
ExpressionUtils.inferNotNull(queryPulledUpPredicates,
+ cascadesContext);
+ Set<Expression> queryUsedNeedRejectNullSlotsViewBased =
nullRejectPredicates.stream()
+ .map(expression ->
TypeUtils.isNotNull(expression).orElse(null))
+ .filter(Objects::nonNull)
+ .map(expr -> ExpressionUtils.replace((Expression) expr,
queryToViewMapping.toSlotReferenceMap()))
+ .collect(Collectors.toSet());
+ // query pulledUp predicates should have null reject predicates and
contains any require noNullable slot
+ return !queryPulledUpPredicates.containsAll(nullRejectPredicates)
+ && requireNoNullableViewSlot.stream().noneMatch(
+ set -> Sets.intersection(set,
queryUsedNeedRejectNullSlotsViewBased).isEmpty());
+ }
+
/**
* Decide the match mode
*
diff --git
a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
index 73c5193f136..0b171742bca 100644
--- a/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
+++ b/regression-test/data/nereids_rules_p0/mv/join/left_outer/outer_join.out
@@ -279,6 +279,14 @@
2023-12-10 2023-12-10 2 4 3
2023-12-10 2023-12-10 2 4 3
+-- !query6_2_before --
+2023-12-10 2023-12-10 2 4 3
+2023-12-10 2023-12-10 2 4 3
+
+-- !query6_2_after --
+2023-12-10 2023-12-10 2 4 3
+2023-12-10 2023-12-10 2 4 3
+
-- !query7_0_before --
3 3 2023-12-11
diff --git
a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
index ffeef5e149f..ef980aa2d9c 100644
---
a/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
+++
b/regression-test/suites/nereids_rules_p0/mv/join/left_outer/outer_join.groovy
@@ -179,7 +179,9 @@ suite("outer_join") {
(2, 4, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-09', '2023-12-09',
'2023-12-10', 'a', 'b', 'yyyyyyyyy'),
(3, 2, 4, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-10', '2023-12-09',
'2023-12-10', 'a', 'b', 'yyyyyyyyy'),
(4, 3, 3, 4, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-12-11', '2023-12-09',
'2023-12-10', 'a', 'b', 'yyyyyyyyy'),
- (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12',
'2023-12-13', 'c', 'd', 'xxxxxxxxx');
+ (5, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12',
'2023-12-13', 'c', 'd', 'xxxxxxxxx'),
+ (6, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12',
'2023-12-13', 'c', 'd', 'xxxxxxxxx'),
+ (7, 2, 3, 6, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-12-12', '2023-12-12',
'2023-12-13', 'c', 'd', 'xxxxxxxxx');
"""
sql """
@@ -490,6 +492,26 @@ suite("outer_join") {
sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_1"""
+ // should compensate predicate o_orderdate = '2023-12-10' on mv
+ def mv6_2 = """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey
+ from lineitem
+ left join (select * from orders where o_orderdate = '2023-12-10' ) t2
+ on lineitem.l_orderkey = t2.o_orderkey;
+ """
+ def query6_2 = """
+ select l_shipdate, o_orderdate, l_partkey, l_suppkey, o_orderkey
+ from lineitem
+ left join orders
+ on lineitem.l_orderkey = orders.o_orderkey
+ where o_orderdate = '2023-12-10' order by 1, 2, 3, 4, 5;
+ """
+ order_qt_query6_2_before "${query6_2}"
+ check_rewrite(mv6_2, query6_2, "mv6_2")
+ order_qt_query6_2_after "${query6_2}"
+ sql """ DROP MATERIALIZED VIEW IF EXISTS mv6_2"""
+
+
// filter inside + left + right
def mv7_0 = "select l_shipdate, o_orderdate, l_partkey, l_suppkey " +
"from lineitem " +
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
