This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ad46e529d8 [feature](Nereids): Infer isNotNull from filter and
eliminate OuterJoin (#16411)
ad46e529d8 is described below
commit ad46e529d84b7d3a6038da338406a515bc745927
Author: jakevin <[email protected]>
AuthorDate: Wed Feb 15 13:33:21 2023 +0800
[feature](Nereids): Infer isNotNull from filter and eliminate OuterJoin
(#16411)
---
.../jobs/batch/NereidsRewriteJobExecutor.java | 4 +
.../org/apache/doris/nereids/rules/RuleType.java | 3 +
.../rules/rewrite/logical/EliminateNotNull.java | 84 +++++++++++++++++++++
.../rules/rewrite/logical/EliminateOuterJoin.java | 67 +++++++---------
.../rules/rewrite/logical/InferFilterNotNull.java | 58 ++++++++++++++
.../nereids/trees/expressions/Expression.java | 10 ++-
.../apache/doris/nereids/util/ExpressionUtils.java | 50 ++++++++++--
.../org/apache/doris/nereids/util/TypeUtils.java | 19 +++++
.../rewrite/logical/EliminateOuterJoinTest.java | 18 ++++-
...erJoinTest.java => InferFilterNotNullTest.java} | 56 ++++++--------
.../apache/doris/nereids/sqltest/InferTest.java | 88 ++++++++++++++++++++++
11 files changed, 374 insertions(+), 83 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
index 1b9d74e244..2a1ec83ed4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
@@ -38,12 +38,14 @@ import
org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimit;
+import org.apache.doris.nereids.rules.rewrite.logical.EliminateNotNull;
import org.apache.doris.nereids.rules.rewrite.logical.EliminateOrderByConstant;
import
org.apache.doris.nereids.rules.rewrite.logical.EliminateUnnecessaryProject;
import
org.apache.doris.nereids.rules.rewrite.logical.ExtractAndNormalizeWindowExpression;
import
org.apache.doris.nereids.rules.rewrite.logical.ExtractFilterFromCrossJoin;
import
org.apache.doris.nereids.rules.rewrite.logical.ExtractSingleTableExpressionFromDisjunction;
import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin;
+import org.apache.doris.nereids.rules.rewrite.logical.InferFilterNotNull;
import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates;
import org.apache.doris.nereids.rules.rewrite.logical.InnerToCrossJoin;
import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown;
@@ -97,6 +99,7 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob {
//e.g. sum(sum(c1)) over(partition by avg(c1))
.add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
.add(topDownBatch(ImmutableList.of(new
CheckAndStandardizeWindowFunctionAndFrame())))
+ .add(topDownBatch(ImmutableList.of(new InferFilterNotNull())))
.add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false))
.add(visitorJob(RuleType.INFER_PREDICATES, new
InferPredicates()))
.add(topDownBatch(ImmutableList.of(new
ExtractFilterFromCrossJoin())))
@@ -112,6 +115,7 @@ public class NereidsRewriteJobExecutor extends
BatchRulesJob {
.add(topDownBatch(ImmutableList.of(new
FindHashConditionForJoin())))
.add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false))
.add(topDownBatch(ImmutableList.of(new InnerToCrossJoin())))
+ .add(topDownBatch(ImmutableList.of(new EliminateNotNull())))
.add(topDownBatch(ImmutableList.of(new EliminateLimit())))
.add(topDownBatch(ImmutableList.of(new EliminateFilter())))
.add(topDownBatch(ImmutableList.of(new
PruneOlapScanPartition())))
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 85d9123ce5..543291af89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -101,6 +101,9 @@ public enum RuleType {
ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
INFER_PREDICATES(RuleTypeClass.REWRITE),
+ INFER_FILTER_NOT_NULL(RuleTypeClass.REWRITE),
+ INFER_JOIN_NOT_NULL(RuleTypeClass.REWRITE),
+ ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE),
// subquery analyze
ANALYZE_FILTER_SUBQUERY(RuleTypeClass.REWRITE),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java
new file mode 100644
index 0000000000..f7a03f5dfe
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.PlanUtils;
+import org.apache.doris.nereids.util.TypeUtils;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Eliminate Predicate `is not null`, like
+ * - redundant `is not null` predicate like `a > 0 and a is not null` -> `a >
0`
+ * - `is not null` predicate is generated by `InferFilterNotNull`
+ */
+public class EliminateNotNull extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalFilter()
+ .when(filter -> filter.getConjuncts().stream().anyMatch(expr ->
expr.isGeneratedIsNotNull))
+ .then(filter -> {
+ // Progress Example: `id > 0 and id is not null and name is
not null(generated)`
+ // predicatesNotContainIsNotNull: `id > 0`
+ // predicatesNotContainIsNotNull infer nonNullable slots: `id`
+ // slotsFromIsNotNull: `id`, `name`
+ // remove `name` (it's generated), remove `id` (because `id >
0` already contains it)
+ Set<Expression> predicatesNotContainIsNotNull =
Sets.newHashSet();
+ List<Slot> slotsFromIsNotNull = Lists.newArrayList();
+ filter.getConjuncts().stream()
+ .filter(expr -> !expr.isGeneratedIsNotNull) // remove
generated `is not null`
+ .forEach(expr -> {
+ Optional<Slot> notNullSlot =
TypeUtils.isNotNull(expr);
+ if (notNullSlot.isPresent()) {
+ slotsFromIsNotNull.add(notNullSlot.get());
+ } else {
+ predicatesNotContainIsNotNull.add(expr);
+ }
+ });
+ Set<Slot> inferNonNotSlots =
ExpressionUtils.inferNotNullSlots(predicatesNotContainIsNotNull);
+
+ Set<Expression> keepIsNotNull = slotsFromIsNotNull.stream()
+ .filter(ExpressionTrait::nullable)
+ .filter(slot -> !inferNonNotSlots.contains(slot))
+ .map(slot -> new Not(new
IsNull(slot))).collect(Collectors.toSet());
+
+ // merge predicatesNotContainIsNotNull and keepIsNotNull into
a new ImmutableSet
+ Set<Expression> newPredicates =
ImmutableSet.<Expression>builder()
+ .addAll(predicatesNotContainIsNotNull)
+ .addAll(keepIsNotNull)
+ .build();
+ return PlanUtils.filterOrSelf(newPredicates, filter.child());
+ }).toRule(RuleType.ELIMINATE_NOT_NULL);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
index 1af7e494e7..c20ed90482 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
@@ -19,20 +19,20 @@ package org.apache.doris.nereids.rules.rewrite.logical;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
-import
org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext;
-import
org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule;
import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.plans.GroupPlan;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.TypeUtils;
-import com.google.common.collect.Maps;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSet.Builder;
-import java.util.Map;
+import java.util.HashSet;
+import java.util.Optional;
import java.util.Set;
/**
@@ -43,24 +43,29 @@ public class EliminateOuterJoin extends
OneRewriteRuleFactory {
@Override
public Rule build() {
return logicalFilter(
- logicalJoin().when(join ->
join.getJoinType().isOuterJoin())
- ).then(filter -> {
- LogicalJoin<GroupPlan, GroupPlan> join = filter.child();
- Set<Expression> conjuncts = filter.getConjuncts();
- Set<Expression> leftPredicates =
ExpressionUtils.extractCoveredConjunction(conjuncts,
- join.left().getOutputSet());
- Set<Expression> rightPredicates =
ExpressionUtils.extractCoveredConjunction(conjuncts,
- join.right().getOutputSet());
- boolean canFilterLeftNull = canFilterNull(leftPredicates);
- boolean canFilterRightNull =
canFilterNull(rightPredicates);
- JoinType newJoinType =
tryEliminateOuterJoin(join.getJoinType(), canFilterLeftNull,
- canFilterRightNull);
- if (newJoinType == join.getJoinType()) {
- return filter;
- } else {
- return
filter.withChildren(join.withJoinType(newJoinType));
- }
- }).toRule(RuleType.ELIMINATE_OUTER_JOIN);
+ logicalJoin().when(join -> join.getJoinType().isOuterJoin())
+ ).then(filter -> {
+ LogicalJoin<GroupPlan, GroupPlan> join = filter.child();
+
+ Builder<Expression> conjunctsBuilder = ImmutableSet.builder();
+ Set<Slot> notNullSlots = new HashSet<>();
+ for (Expression predicate : filter.getConjuncts()) {
+ Optional<Slot> notNullSlot = TypeUtils.isNotNull(predicate);
+ if (notNullSlot.isPresent()) {
+ notNullSlots.add(notNullSlot.get());
+ } else {
+ conjunctsBuilder.add(predicate);
+ }
+ }
+ boolean canFilterLeftNull =
ExpressionUtils.isIntersecting(join.left().getOutputSet(), notNullSlots);
+ boolean canFilterRightNull =
ExpressionUtils.isIntersecting(join.right().getOutputSet(), notNullSlots);
+ if (!canFilterRightNull && !canFilterLeftNull) {
+ return null;
+ }
+
+ JoinType newJoinType = tryEliminateOuterJoin(join.getJoinType(),
canFilterLeftNull, canFilterRightNull);
+ return filter.withChildren(join.withJoinType(newJoinType));
+ }).toRule(RuleType.ELIMINATE_OUTER_JOIN);
}
private JoinType tryEliminateOuterJoin(JoinType joinType, boolean
canFilterLeftNull, boolean canFilterRightNull) {
@@ -81,18 +86,4 @@ public class EliminateOuterJoin extends
OneRewriteRuleFactory {
}
return joinType;
}
-
- private boolean canFilterNull(Set<Expression> predicates) {
- Literal nullLiteral = Literal.of(null);
- for (Expression predicate : predicates) {
- Map<Expression, Expression> replaceMap = Maps.newHashMap();
- predicate.getInputSlots().forEach(slot -> replaceMap.put(slot,
nullLiteral));
- Expression evalExpr =
FoldConstantRule.INSTANCE.rewrite(ExpressionUtils.replace(predicate,
replaceMap),
- new ExpressionRewriteContext(null));
- if (nullLiteral.equals(evalExpr) ||
BooleanLiteral.FALSE.equals(evalExpr)) {
- return true;
- }
- }
- return false;
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java
new file mode 100644
index 0000000000..02fac54ff3
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.PlanUtils;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSet.Builder;
+
+import java.util.Set;
+
+/**
+ * InferNotNull from Filter.
+ * Like:
+ * filter a + b > 0
+ * ->
+ * filter a + b > 0 and a is not null and b is not null
+ * <p>
+ * This rule will cooperate with PushDownFilter and EliminateNotNull.
+ */
+public class InferFilterNotNull extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalFilter()
+ .when(filter -> filter.getConjuncts().stream().noneMatch(expr ->
expr.isGeneratedIsNotNull))
+ .then(filter -> {
+ Set<Expression> predicates = filter.getConjuncts();
+ Set<Expression> isNotNull =
ExpressionUtils.inferNotNull(predicates);
+ if (isNotNull.isEmpty() || predicates.containsAll(isNotNull)) {
+ return null;
+ }
+ Builder<Expression> builder =
ImmutableSet.<Expression>builder()
+ .addAll(predicates)
+ .addAll(isNotNull);
+ return PlanUtils.filter(builder.build(), filter.child()).get();
+ }).toRule(RuleType.INFER_FILTER_NOT_NULL);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
index a45af25e55..fe5488db4e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -45,8 +45,8 @@ import java.util.stream.Collectors;
* Abstract class for all Expression in Nereids.
*/
public abstract class Expression extends AbstractTreeNode<Expression>
implements ExpressionTrait {
-
- private static final String INPUT_CHECK_ERROR_MESSAGE = "argument %d
requires %s type, however '%s' is of %s type";
+ // Mask this expression is generated by rule, should be removed.
+ public boolean isGeneratedIsNotNull = false;
public Expression(Expression... children) {
super(children);
@@ -89,7 +89,7 @@ public abstract class Expression extends
AbstractTreeNode<Expression> implements
Expression input = inputs.get(i);
AbstractDataType inputType = inputTypes.get(i);
if (!inputType.acceptsType(input.getDataType())) {
- errorMessages.add(String.format(INPUT_CHECK_ERROR_MESSAGE,
+ errorMessages.add(String.format("argument %d requires %s type,
however '%s' is of %s type",
i + 1, inputType.simpleString(), input.toSql(),
input.getDataType().simpleString()));
}
}
@@ -117,7 +117,9 @@ public abstract class Expression extends
AbstractTreeNode<Expression> implements
}
public final Expression withChildren(Expression... children) {
- return withChildren(ImmutableList.copyOf(children));
+ Expression expr = withChildren(ImmutableList.copyOf(children));
+ expr.isGeneratedIsNotNull = this.isGeneratedIsNotNull;
+ return expr;
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 19e17986ab..f067c9dba8 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.util;
+import
org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext;
+import
org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule;
import org.apache.doris.nereids.trees.TreeNode;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.Cast;
@@ -45,12 +47,14 @@ import com.google.common.collect.Sets;
import java.util.Arrays;
import java.util.Collection;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
import java.util.function.Function;
+import java.util.stream.Collectors;
/**
* Expression rewrite helper class.
@@ -325,6 +329,7 @@ public class ExpressionUtils {
/**
* merge arguments into an expression array
+ *
* @param arguments instance of Expression or Expression Array
* @return Expression Array
*/
@@ -361,16 +366,49 @@ public class ExpressionUtils {
}
/**
- * extract the predicate that is covered by `slots`
+ * infer notNulls slot from predicate
*/
- public static Set<Expression> extractCoveredConjunction(Set<Expression>
predicates, Set<Slot> slots) {
- Set<Expression> coveredPredicates = Sets.newHashSet();
+ public static Set<Slot> inferNotNullSlots(Set<Expression> predicates) {
+ Literal nullLiteral = Literal.of(null);
+ Set<Slot> notNullSlots = Sets.newHashSet();
for (Expression predicate : predicates) {
- if (slots.containsAll(predicate.getInputSlots())) {
- coveredPredicates.add(predicate);
+ for (Slot slot : predicate.getInputSlots()) {
+ Map<Expression, Expression> replaceMap = new HashMap<>();
+ replaceMap.put(slot, nullLiteral);
+ Expression evalExpr = FoldConstantRule.INSTANCE.rewrite(
+ ExpressionUtils.replace(predicate, replaceMap),
+ new ExpressionRewriteContext(null));
+ if (nullLiteral.equals(evalExpr) ||
BooleanLiteral.FALSE.equals(evalExpr)) {
+ notNullSlots.add(slot);
+ }
}
}
- return coveredPredicates;
+ return notNullSlots;
+ }
+
+ /**
+ * infer notNulls slot from predicate
+ */
+ public static Set<Expression> inferNotNull(Set<Expression> predicates) {
+ return inferNotNullSlots(predicates).stream()
+ .map(slot -> {
+ Not isNotNull = new Not(new IsNull(slot));
+ isNotNull.isGeneratedIsNotNull = true;
+ return isNotNull;
+ }).collect(Collectors.toSet());
+ }
+
+ /**
+ * infer notNulls slot from predicate but these slots must be in the given
slots.
+ */
+ public static Set<Expression> inferNotNull(Set<Expression> predicates,
Set<Slot> slots) {
+ return inferNotNullSlots(predicates).stream()
+ .filter(slots::contains)
+ .map(slot -> {
+ Not isNotNull = new Not(new IsNull(slot));
+ isNotNull.isGeneratedIsNotNull = true;
+ return isNotNull;
+ }).collect(Collectors.toSet());
}
public static <E extends Expression> List<E> flatExpressions(List<List<E>>
expressions) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
index 318f1155bb..9e147d30af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
@@ -20,9 +20,15 @@ package org.apache.doris.nereids.util;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.Divide;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
import org.apache.doris.nereids.trees.expressions.Multiply;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.Subtract;
+import java.util.Optional;
+
/**
* Judgment expression type.
*/
@@ -52,4 +58,17 @@ public class TypeUtils {
return expr instanceof Multiply;
}
+ /**
+ * Judge whether the expression is `is not null`.
+ *
+ * @return Optional.empty() if the expression is `is not null`, otherwise
return slot.
+ */
+ public static Optional<Slot> isNotNull(Expression expr) {
+ if (expr instanceof Not && ((Not) expr).child() instanceof IsNull
+ && ((IsNull) ((Not) expr).child()).child() instanceof
SlotReference) {
+ return Optional.of(((SlotReference) ((IsNull) ((Not)
expr).child()).child()));
+ } else {
+ return Optional.empty();
+ }
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
index 317d7f8f96..4777251c56 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
@@ -32,6 +32,8 @@ import org.apache.doris.nereids.util.PlanConstructor;
import org.junit.jupiter.api.Test;
+import java.util.Objects;
+
class EliminateOuterJoinTest implements PatternMatchSupported {
private final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
private final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
@@ -44,11 +46,14 @@ class EliminateOuterJoinTest implements
PatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new InferFilterNotNull())
.applyTopDown(new EliminateOuterJoin())
+ .applyTopDown(new EliminateNotNull())
+ .printlnTree()
.matchesFromRoot(
logicalFilter(
logicalJoin().when(join ->
join.getJoinType().isInnerJoin())
- )
+ ).when(filter -> filter.getConjuncts().size() == 1)
);
}
@@ -60,11 +65,15 @@ class EliminateOuterJoinTest implements
PatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new InferFilterNotNull())
.applyTopDown(new EliminateOuterJoin())
+ .applyTopDown(new EliminateNotNull())
+ .printlnTree()
.matchesFromRoot(
logicalFilter(
logicalJoin().when(join ->
join.getJoinType().isInnerJoin())
- )
+ ).when(filter -> filter.getConjuncts().size() == 1)
+ .when(filter ->
Objects.equals(filter.getConjuncts().toString(), "[(id#0 > 1)]"))
);
}
@@ -78,11 +87,14 @@ class EliminateOuterJoinTest implements
PatternMatchSupported {
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new InferFilterNotNull())
.applyTopDown(new EliminateOuterJoin())
+ .applyTopDown(new EliminateNotNull())
+ .printlnTree()
.matchesFromRoot(
logicalFilter(
logicalJoin().when(join ->
join.getJoinType().isInnerJoin())
- )
+ ).when(filter -> filter.getConjuncts().size() == 2)
);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java
similarity index 52%
copy from
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
copy to
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java
index 317d7f8f96..7c0d2623cc 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java
@@ -17,11 +17,11 @@
package org.apache.doris.nereids.rules.rewrite.logical;
-import org.apache.doris.common.Pair;
-import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.Add;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
-import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.util.LogicalPlanBuilder;
@@ -32,57 +32,49 @@ import org.apache.doris.nereids.util.PlanConstructor;
import org.junit.jupiter.api.Test;
-class EliminateOuterJoinTest implements PatternMatchSupported {
+class InferFilterNotNullTest implements PatternMatchSupported {
private final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
- private final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
@Test
- void testEliminateLeft() {
+ void testInfer() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0)) //
t1.id = t2.id
- .filter(new GreaterThan(scan2.getOutput().get(0),
Literal.of(1)))
+ .filter(new EqualTo(new Add(scan1.getOutput().get(0),
scan1.getOutput().get(1)), Literal.of(1)))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new EliminateOuterJoin())
- .matchesFromRoot(
- logicalFilter(
- logicalJoin().when(join ->
join.getJoinType().isInnerJoin())
- )
+ .applyTopDown(new InferFilterNotNull())
+ .matches(
+ // LogicalFilter ( predicates=((((id#0 + name#1) = 1)
AND ( not id IS NULL)) AND ( not name IS NULL)) )
+ logicalFilter().when(filter ->
filter.getConjuncts().size() == 3)
);
}
@Test
- void testEliminateRight() {
+ void testInferFail() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0)) //
t1.id = t2.id
- .filter(new GreaterThan(scan1.getOutput().get(0),
Literal.of(1)))
+ .filter(new IsNull(scan1.getOutput().get(0)))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new EliminateOuterJoin())
- .matchesFromRoot(
- logicalFilter(
- logicalJoin().when(join ->
join.getJoinType().isInnerJoin())
- )
+ .applyTopDown(new InferFilterNotNull())
+ .matches(
+ // LogicalFilter ( predicates=id IS NULL )
+ logicalFilter().when(filter ->
filter.getConjuncts().size() == 1)
);
}
@Test
- void testEliminateBoth() {
+ void testInferFailOr() {
LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(0, 0)) //
t1.id = t2.id
- .filter(new And(
- new GreaterThan(scan2.getOutput().get(0),
Literal.of(1)),
- new GreaterThan(scan1.getOutput().get(0),
Literal.of(1))))
+ .filter(new Or(new IsNull(scan1.getOutput().get(0)),
+ new EqualTo(scan1.getOutput().get(0), Literal.of(1))))
.build();
PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new EliminateOuterJoin())
- .matchesFromRoot(
- logicalFilter(
- logicalJoin().when(join ->
join.getJoinType().isInnerJoin())
- )
+ .applyTopDown(new InferFilterNotNull())
+ .matches(
+ // LogicalFilter ( predicates=(id IS NULL OR (id#0 =
1)) )
+ logicalFilter().when(filter ->
filter.getConjuncts().size() == 1)
);
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
new file mode 100644
index 0000000000..596a08116d
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.sqltest;
+
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class InferTest extends SqlTestBase {
+ @Test
+ void testInferNotNullAndInferPredicates() {
+ // Test InferNotNull, EliminateOuter, InferPredicate together
+ String sql = "select * from T1 left outer join T2 on T1.id = T2.id
where T2.id = 4";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matchesFromRoot(
+ innerLogicalJoin(
+ logicalFilter().when(f ->
f.getPredicate().toString().equals("(id#0 = 4)")),
+ logicalFilter().when(f ->
f.getPredicate().toString().equals("(id#2 = 4)"))
+ )
+ );
+ }
+
+ @Test
+ void testInferNotNullFromFilterAndEliminateOuter2() {
+ String sql
+ = "select * from T1 right outer join T2 on T1.id = T2.id where
T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .printlnTree()
+ .matchesFromRoot(
+ innerLogicalJoin(
+ logicalFilter().when(
+ f ->
f.getPredicate().toString().equals("((id#0 = 4) OR (id#0 > 4))")),
+ logicalOlapScan()
+ )
+ );
+ }
+
+ @Test
+ void testInferNotNullFromFilterAndEliminateOuter3() {
+ String sql
+ = "select * from T1 full outer join T2 on T1.id = T2.id where
T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .matchesFromRoot(
+ logicalFilter(
+ leftOuterLogicalJoin(
+ logicalFilter().when(
+ f ->
f.getPredicate().toString().equals("((id#0 = 4) OR (id#0 > 4))")),
+ logicalOlapScan()
+ )
+ ).when(f -> f.getPredicate().toString()
+ .equals("((id#0 = 4) OR ((id#0 > 4) AND score
IS NULL))"))
+ );
+ }
+
+ @Test
+ @Disabled
+ void testInferNotNullFromJoinAndEliminateOuter() {
+ String sql
+ = "select * from (select T1.id from T1 left outer join T2 on
T1.id = T2.id) T1 left semi join T3 on T1.id = T3.id";
+ PlanChecker.from(connectContext)
+ .analyze(sql)
+ .printlnTree()
+ .rewrite()
+ .printlnTree();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]