This is an automated email from the ASF dual-hosted git repository.

jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new ad46e529d8 [feature](Nereids): Infer isNotNull from filter and 
eliminate OuterJoin (#16411)
ad46e529d8 is described below

commit ad46e529d84b7d3a6038da338406a515bc745927
Author: jakevin <[email protected]>
AuthorDate: Wed Feb 15 13:33:21 2023 +0800

    [feature](Nereids): Infer isNotNull from filter and eliminate OuterJoin 
(#16411)
---
 .../jobs/batch/NereidsRewriteJobExecutor.java      |  4 +
 .../org/apache/doris/nereids/rules/RuleType.java   |  3 +
 .../rules/rewrite/logical/EliminateNotNull.java    | 84 +++++++++++++++++++++
 .../rules/rewrite/logical/EliminateOuterJoin.java  | 67 +++++++---------
 .../rules/rewrite/logical/InferFilterNotNull.java  | 58 ++++++++++++++
 .../nereids/trees/expressions/Expression.java      | 10 ++-
 .../apache/doris/nereids/util/ExpressionUtils.java | 50 ++++++++++--
 .../org/apache/doris/nereids/util/TypeUtils.java   | 19 +++++
 .../rewrite/logical/EliminateOuterJoinTest.java    | 18 ++++-
 ...erJoinTest.java => InferFilterNotNullTest.java} | 56 ++++++--------
 .../apache/doris/nereids/sqltest/InferTest.java    | 88 ++++++++++++++++++++++
 11 files changed, 374 insertions(+), 83 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
index 1b9d74e244..2a1ec83ed4 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriteJobExecutor.java
@@ -38,12 +38,14 @@ import 
org.apache.doris.nereids.rules.rewrite.logical.EliminateAggregate;
 import org.apache.doris.nereids.rules.rewrite.logical.EliminateFilter;
 import org.apache.doris.nereids.rules.rewrite.logical.EliminateGroupByConstant;
 import org.apache.doris.nereids.rules.rewrite.logical.EliminateLimit;
+import org.apache.doris.nereids.rules.rewrite.logical.EliminateNotNull;
 import org.apache.doris.nereids.rules.rewrite.logical.EliminateOrderByConstant;
 import 
org.apache.doris.nereids.rules.rewrite.logical.EliminateUnnecessaryProject;
 import 
org.apache.doris.nereids.rules.rewrite.logical.ExtractAndNormalizeWindowExpression;
 import 
org.apache.doris.nereids.rules.rewrite.logical.ExtractFilterFromCrossJoin;
 import 
org.apache.doris.nereids.rules.rewrite.logical.ExtractSingleTableExpressionFromDisjunction;
 import org.apache.doris.nereids.rules.rewrite.logical.FindHashConditionForJoin;
+import org.apache.doris.nereids.rules.rewrite.logical.InferFilterNotNull;
 import org.apache.doris.nereids.rules.rewrite.logical.InferPredicates;
 import org.apache.doris.nereids.rules.rewrite.logical.InnerToCrossJoin;
 import org.apache.doris.nereids.rules.rewrite.logical.LimitPushDown;
@@ -97,6 +99,7 @@ public class NereidsRewriteJobExecutor extends BatchRulesJob {
                 //e.g. sum(sum(c1)) over(partition by avg(c1))
                 .add(topDownBatch(ImmutableList.of(new NormalizeAggregate())))
                 .add(topDownBatch(ImmutableList.of(new 
CheckAndStandardizeWindowFunctionAndFrame())))
+                .add(topDownBatch(ImmutableList.of(new InferFilterNotNull())))
                 .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false))
                 .add(visitorJob(RuleType.INFER_PREDICATES, new 
InferPredicates()))
                 .add(topDownBatch(ImmutableList.of(new 
ExtractFilterFromCrossJoin())))
@@ -112,6 +115,7 @@ public class NereidsRewriteJobExecutor extends 
BatchRulesJob {
                 .add(topDownBatch(ImmutableList.of(new 
FindHashConditionForJoin())))
                 .add(topDownBatch(RuleSet.PUSH_DOWN_FILTERS, false))
                 .add(topDownBatch(ImmutableList.of(new InnerToCrossJoin())))
+                .add(topDownBatch(ImmutableList.of(new EliminateNotNull())))
                 .add(topDownBatch(ImmutableList.of(new EliminateLimit())))
                 .add(topDownBatch(ImmutableList.of(new EliminateFilter())))
                 .add(topDownBatch(ImmutableList.of(new 
PruneOlapScanPartition())))
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 85d9123ce5..543291af89 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -101,6 +101,9 @@ public enum RuleType {
     ELIMINATE_GROUP_BY_CONSTANT(RuleTypeClass.REWRITE),
     ELIMINATE_ORDER_BY_CONSTANT(RuleTypeClass.REWRITE),
     INFER_PREDICATES(RuleTypeClass.REWRITE),
+    INFER_FILTER_NOT_NULL(RuleTypeClass.REWRITE),
+    INFER_JOIN_NOT_NULL(RuleTypeClass.REWRITE),
+    ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE),
 
     // subquery analyze
     ANALYZE_FILTER_SUBQUERY(RuleTypeClass.REWRITE),
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java
new file mode 100644
index 0000000000..f7a03f5dfe
--- /dev/null
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateNotNull.java
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.PlanUtils;
+import org.apache.doris.nereids.util.TypeUtils;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Eliminate Predicate `is not null`, like
+ * - redundant `is not null` predicate like `a > 0 and a is not null` -> `a > 
0`
+ * - `is not null` predicate is generated by `InferFilterNotNull`
+ */
+public class EliminateNotNull extends OneRewriteRuleFactory {
+    @Override
+    public Rule build() {
+        return logicalFilter()
+            .when(filter -> filter.getConjuncts().stream().anyMatch(expr -> 
expr.isGeneratedIsNotNull))
+            .then(filter -> {
+                // Progress Example: `id > 0 and id is not null and name is 
not null(generated)`
+                // predicatesNotContainIsNotNull: `id > 0`
+                // predicatesNotContainIsNotNull infer nonNullable slots: `id`
+                // slotsFromIsNotNull: `id`, `name`
+                // remove `name` (it's generated), remove `id` (because `id > 
0` already contains it)
+                Set<Expression> predicatesNotContainIsNotNull = 
Sets.newHashSet();
+                List<Slot> slotsFromIsNotNull = Lists.newArrayList();
+                filter.getConjuncts().stream()
+                        .filter(expr -> !expr.isGeneratedIsNotNull) // remove 
generated `is not null`
+                        .forEach(expr -> {
+                            Optional<Slot> notNullSlot = 
TypeUtils.isNotNull(expr);
+                            if (notNullSlot.isPresent()) {
+                                slotsFromIsNotNull.add(notNullSlot.get());
+                            } else {
+                                predicatesNotContainIsNotNull.add(expr);
+                            }
+                        });
+                Set<Slot> inferNonNotSlots = 
ExpressionUtils.inferNotNullSlots(predicatesNotContainIsNotNull);
+
+                Set<Expression> keepIsNotNull = slotsFromIsNotNull.stream()
+                        .filter(ExpressionTrait::nullable)
+                        .filter(slot -> !inferNonNotSlots.contains(slot))
+                        .map(slot -> new Not(new 
IsNull(slot))).collect(Collectors.toSet());
+
+                // merge predicatesNotContainIsNotNull and keepIsNotNull into 
a new ImmutableSet
+                Set<Expression> newPredicates = 
ImmutableSet.<Expression>builder()
+                        .addAll(predicatesNotContainIsNotNull)
+                        .addAll(keepIsNotNull)
+                        .build();
+                return PlanUtils.filterOrSelf(newPredicates, filter.child());
+            }).toRule(RuleType.ELIMINATE_NOT_NULL);
+    }
+}
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
index 1af7e494e7..c20ed90482 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoin.java
@@ -19,20 +19,20 @@ package org.apache.doris.nereids.rules.rewrite.logical;
 
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
-import 
org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext;
-import 
org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule;
 import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
 import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
-import org.apache.doris.nereids.trees.expressions.literal.Literal;
+import org.apache.doris.nereids.trees.expressions.Slot;
 import org.apache.doris.nereids.trees.plans.GroupPlan;
 import org.apache.doris.nereids.trees.plans.JoinType;
 import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
 import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.TypeUtils;
 
-import com.google.common.collect.Maps;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSet.Builder;
 
-import java.util.Map;
+import java.util.HashSet;
+import java.util.Optional;
 import java.util.Set;
 
 /**
@@ -43,24 +43,29 @@ public class EliminateOuterJoin extends 
OneRewriteRuleFactory {
     @Override
     public Rule build() {
         return logicalFilter(
-                    logicalJoin().when(join -> 
join.getJoinType().isOuterJoin())
-                ).then(filter -> {
-                    LogicalJoin<GroupPlan, GroupPlan> join = filter.child();
-                    Set<Expression> conjuncts = filter.getConjuncts();
-                    Set<Expression> leftPredicates = 
ExpressionUtils.extractCoveredConjunction(conjuncts,
-                            join.left().getOutputSet());
-                    Set<Expression> rightPredicates = 
ExpressionUtils.extractCoveredConjunction(conjuncts,
-                            join.right().getOutputSet());
-                    boolean canFilterLeftNull = canFilterNull(leftPredicates);
-                    boolean canFilterRightNull = 
canFilterNull(rightPredicates);
-                    JoinType newJoinType = 
tryEliminateOuterJoin(join.getJoinType(), canFilterLeftNull,
-                            canFilterRightNull);
-                    if (newJoinType == join.getJoinType()) {
-                        return filter;
-                    } else {
-                        return 
filter.withChildren(join.withJoinType(newJoinType));
-                    }
-                }).toRule(RuleType.ELIMINATE_OUTER_JOIN);
+                logicalJoin().when(join -> join.getJoinType().isOuterJoin())
+        ).then(filter -> {
+            LogicalJoin<GroupPlan, GroupPlan> join = filter.child();
+
+            Builder<Expression> conjunctsBuilder = ImmutableSet.builder();
+            Set<Slot> notNullSlots = new HashSet<>();
+            for (Expression predicate : filter.getConjuncts()) {
+                Optional<Slot> notNullSlot = TypeUtils.isNotNull(predicate);
+                if (notNullSlot.isPresent()) {
+                    notNullSlots.add(notNullSlot.get());
+                } else {
+                    conjunctsBuilder.add(predicate);
+                }
+            }
+            boolean canFilterLeftNull = 
ExpressionUtils.isIntersecting(join.left().getOutputSet(), notNullSlots);
+            boolean canFilterRightNull = 
ExpressionUtils.isIntersecting(join.right().getOutputSet(), notNullSlots);
+            if (!canFilterRightNull && !canFilterLeftNull) {
+                return null;
+            }
+
+            JoinType newJoinType = tryEliminateOuterJoin(join.getJoinType(), 
canFilterLeftNull, canFilterRightNull);
+            return filter.withChildren(join.withJoinType(newJoinType));
+        }).toRule(RuleType.ELIMINATE_OUTER_JOIN);
     }
 
     private JoinType tryEliminateOuterJoin(JoinType joinType, boolean 
canFilterLeftNull, boolean canFilterRightNull) {
@@ -81,18 +86,4 @@ public class EliminateOuterJoin extends 
OneRewriteRuleFactory {
         }
         return joinType;
     }
-
-    private boolean canFilterNull(Set<Expression> predicates) {
-        Literal nullLiteral = Literal.of(null);
-        for (Expression predicate : predicates) {
-            Map<Expression, Expression> replaceMap = Maps.newHashMap();
-            predicate.getInputSlots().forEach(slot -> replaceMap.put(slot, 
nullLiteral));
-            Expression evalExpr = 
FoldConstantRule.INSTANCE.rewrite(ExpressionUtils.replace(predicate, 
replaceMap),
-                    new ExpressionRewriteContext(null));
-            if (nullLiteral.equals(evalExpr) || 
BooleanLiteral.FALSE.equals(evalExpr)) {
-                return true;
-            }
-        }
-        return false;
-    }
 }
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java
new file mode 100644
index 0000000000..02fac54ff3
--- /dev/null
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNull.java
@@ -0,0 +1,58 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.util.ExpressionUtils;
+import org.apache.doris.nereids.util.PlanUtils;
+
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.ImmutableSet.Builder;
+
+import java.util.Set;
+
+/**
+ * InferNotNull from Filter.
+ * Like:
+ * filter a + b > 0
+ * ->
+ * filter a + b > 0 and a is not null and b is not null
+ * <p>
+ * This rule will cooperate with PushDownFilter and EliminateNotNull.
+ */
+public class InferFilterNotNull extends OneRewriteRuleFactory {
+    @Override
+    public Rule build() {
+        return logicalFilter()
+            .when(filter -> filter.getConjuncts().stream().noneMatch(expr -> 
expr.isGeneratedIsNotNull))
+            .then(filter -> {
+                Set<Expression> predicates = filter.getConjuncts();
+                Set<Expression> isNotNull = 
ExpressionUtils.inferNotNull(predicates);
+                if (isNotNull.isEmpty() || predicates.containsAll(isNotNull)) {
+                    return null;
+                }
+                Builder<Expression> builder = 
ImmutableSet.<Expression>builder()
+                        .addAll(predicates)
+                        .addAll(isNotNull);
+                return PlanUtils.filter(builder.build(), filter.child()).get();
+            }).toRule(RuleType.INFER_FILTER_NOT_NULL);
+    }
+}
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
index a45af25e55..fe5488db4e 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/Expression.java
@@ -45,8 +45,8 @@ import java.util.stream.Collectors;
  * Abstract class for all Expression in Nereids.
  */
 public abstract class Expression extends AbstractTreeNode<Expression> 
implements ExpressionTrait {
-
-    private static final String INPUT_CHECK_ERROR_MESSAGE = "argument %d 
requires %s type, however '%s' is of %s type";
+    // Mask this expression is generated by rule, should be removed.
+    public boolean isGeneratedIsNotNull = false;
 
     public Expression(Expression... children) {
         super(children);
@@ -89,7 +89,7 @@ public abstract class Expression extends 
AbstractTreeNode<Expression> implements
             Expression input = inputs.get(i);
             AbstractDataType inputType = inputTypes.get(i);
             if (!inputType.acceptsType(input.getDataType())) {
-                errorMessages.add(String.format(INPUT_CHECK_ERROR_MESSAGE,
+                errorMessages.add(String.format("argument %d requires %s type, 
however '%s' is of %s type",
                         i + 1, inputType.simpleString(), input.toSql(), 
input.getDataType().simpleString()));
             }
         }
@@ -117,7 +117,9 @@ public abstract class Expression extends 
AbstractTreeNode<Expression> implements
     }
 
     public final Expression withChildren(Expression... children) {
-        return withChildren(ImmutableList.copyOf(children));
+        Expression expr = withChildren(ImmutableList.copyOf(children));
+        expr.isGeneratedIsNotNull = this.isGeneratedIsNotNull;
+        return expr;
     }
 
     /**
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 19e17986ab..f067c9dba8 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -17,6 +17,8 @@
 
 package org.apache.doris.nereids.util;
 
+import 
org.apache.doris.nereids.rules.expression.rewrite.ExpressionRewriteContext;
+import 
org.apache.doris.nereids.rules.expression.rewrite.rules.FoldConstantRule;
 import org.apache.doris.nereids.trees.TreeNode;
 import org.apache.doris.nereids.trees.expressions.And;
 import org.apache.doris.nereids.trees.expressions.Cast;
@@ -45,12 +47,14 @@ import com.google.common.collect.Sets;
 
 import java.util.Arrays;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Function;
+import java.util.stream.Collectors;
 
 /**
  * Expression rewrite helper class.
@@ -325,6 +329,7 @@ public class ExpressionUtils {
 
     /**
      * merge arguments into an expression array
+     *
      * @param arguments instance of Expression or Expression Array
      * @return Expression Array
      */
@@ -361,16 +366,49 @@ public class ExpressionUtils {
     }
 
     /**
-     * extract the predicate that is covered by `slots`
+     * infer notNulls slot from predicate
      */
-    public static Set<Expression> extractCoveredConjunction(Set<Expression> 
predicates, Set<Slot> slots) {
-        Set<Expression> coveredPredicates = Sets.newHashSet();
+    public static Set<Slot> inferNotNullSlots(Set<Expression> predicates) {
+        Literal nullLiteral = Literal.of(null);
+        Set<Slot> notNullSlots = Sets.newHashSet();
         for (Expression predicate : predicates) {
-            if (slots.containsAll(predicate.getInputSlots())) {
-                coveredPredicates.add(predicate);
+            for (Slot slot : predicate.getInputSlots()) {
+                Map<Expression, Expression> replaceMap = new HashMap<>();
+                replaceMap.put(slot, nullLiteral);
+                Expression evalExpr = FoldConstantRule.INSTANCE.rewrite(
+                        ExpressionUtils.replace(predicate, replaceMap),
+                        new ExpressionRewriteContext(null));
+                if (nullLiteral.equals(evalExpr) || 
BooleanLiteral.FALSE.equals(evalExpr)) {
+                    notNullSlots.add(slot);
+                }
             }
         }
-        return coveredPredicates;
+        return notNullSlots;
+    }
+
+    /**
+     * infer notNulls slot from predicate
+     */
+    public static Set<Expression> inferNotNull(Set<Expression> predicates) {
+        return inferNotNullSlots(predicates).stream()
+                .map(slot -> {
+                    Not isNotNull = new Not(new IsNull(slot));
+                    isNotNull.isGeneratedIsNotNull = true;
+                    return isNotNull;
+                }).collect(Collectors.toSet());
+    }
+
+    /**
+     * infer notNulls slot from predicate but these slots must be in the given 
slots.
+     */
+    public static Set<Expression> inferNotNull(Set<Expression> predicates, 
Set<Slot> slots) {
+        return inferNotNullSlots(predicates).stream()
+                .filter(slots::contains)
+                .map(slot -> {
+                    Not isNotNull = new Not(new IsNull(slot));
+                    isNotNull.isGeneratedIsNotNull = true;
+                    return isNotNull;
+                }).collect(Collectors.toSet());
     }
 
     public static <E extends Expression> List<E> flatExpressions(List<List<E>> 
expressions) {
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
index 318f1155bb..9e147d30af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/TypeUtils.java
@@ -20,9 +20,15 @@ package org.apache.doris.nereids.util;
 import org.apache.doris.nereids.trees.expressions.Add;
 import org.apache.doris.nereids.trees.expressions.Divide;
 import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.IsNull;
 import org.apache.doris.nereids.trees.expressions.Multiply;
+import org.apache.doris.nereids.trees.expressions.Not;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
 import org.apache.doris.nereids.trees.expressions.Subtract;
 
+import java.util.Optional;
+
 /**
  * Judgment expression type.
  */
@@ -52,4 +58,17 @@ public class TypeUtils {
         return expr instanceof Multiply;
     }
 
+    /**
+     * Judge whether the expression is `is not null`.
+     *
+     * @return Optional.empty() if the expression is `is not null`, otherwise 
return slot.
+     */
+    public static Optional<Slot> isNotNull(Expression expr) {
+        if (expr instanceof Not && ((Not) expr).child() instanceof IsNull
+                && ((IsNull) ((Not) expr).child()).child() instanceof 
SlotReference) {
+            return Optional.of(((SlotReference) ((IsNull) ((Not) 
expr).child()).child()));
+        } else {
+            return Optional.empty();
+        }
+    }
 }
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
index 317d7f8f96..4777251c56 100644
--- 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
+++ 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
@@ -32,6 +32,8 @@ import org.apache.doris.nereids.util.PlanConstructor;
 
 import org.junit.jupiter.api.Test;
 
+import java.util.Objects;
+
 class EliminateOuterJoinTest implements PatternMatchSupported {
     private final LogicalOlapScan scan1 = 
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
     private final LogicalOlapScan scan2 = 
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
@@ -44,11 +46,14 @@ class EliminateOuterJoinTest implements 
PatternMatchSupported {
                 .build();
 
         PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+                .applyTopDown(new InferFilterNotNull())
                 .applyTopDown(new EliminateOuterJoin())
+                .applyTopDown(new EliminateNotNull())
+                .printlnTree()
                 .matchesFromRoot(
                         logicalFilter(
                                 logicalJoin().when(join -> 
join.getJoinType().isInnerJoin())
-                        )
+                        ).when(filter -> filter.getConjuncts().size() == 1)
                 );
     }
 
@@ -60,11 +65,15 @@ class EliminateOuterJoinTest implements 
PatternMatchSupported {
                 .build();
 
         PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+                .applyTopDown(new InferFilterNotNull())
                 .applyTopDown(new EliminateOuterJoin())
+                .applyTopDown(new EliminateNotNull())
+                .printlnTree()
                 .matchesFromRoot(
                         logicalFilter(
                                 logicalJoin().when(join -> 
join.getJoinType().isInnerJoin())
-                        )
+                        ).when(filter -> filter.getConjuncts().size() == 1)
+                                .when(filter -> 
Objects.equals(filter.getConjuncts().toString(), "[(id#0 > 1)]"))
                 );
     }
 
@@ -78,11 +87,14 @@ class EliminateOuterJoinTest implements 
PatternMatchSupported {
                 .build();
 
         PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+                .applyTopDown(new InferFilterNotNull())
                 .applyTopDown(new EliminateOuterJoin())
+                .applyTopDown(new EliminateNotNull())
+                .printlnTree()
                 .matchesFromRoot(
                         logicalFilter(
                                 logicalJoin().when(join -> 
join.getJoinType().isInnerJoin())
-                        )
+                        ).when(filter -> filter.getConjuncts().size() == 2)
                 );
     }
 }
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java
similarity index 52%
copy from 
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
copy to 
fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java
index 317d7f8f96..7c0d2623cc 100644
--- 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/EliminateOuterJoinTest.java
+++ 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/InferFilterNotNullTest.java
@@ -17,11 +17,11 @@
 
 package org.apache.doris.nereids.rules.rewrite.logical;
 
-import org.apache.doris.common.Pair;
-import org.apache.doris.nereids.trees.expressions.And;
-import org.apache.doris.nereids.trees.expressions.GreaterThan;
+import org.apache.doris.nereids.trees.expressions.Add;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.IsNull;
+import org.apache.doris.nereids.trees.expressions.Or;
 import org.apache.doris.nereids.trees.expressions.literal.Literal;
-import org.apache.doris.nereids.trees.plans.JoinType;
 import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
 import org.apache.doris.nereids.util.LogicalPlanBuilder;
@@ -32,57 +32,49 @@ import org.apache.doris.nereids.util.PlanConstructor;
 
 import org.junit.jupiter.api.Test;
 
-class EliminateOuterJoinTest implements PatternMatchSupported {
+class InferFilterNotNullTest implements PatternMatchSupported {
     private final LogicalOlapScan scan1 = 
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
-    private final LogicalOlapScan scan2 = 
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
 
     @Test
-    void testEliminateLeft() {
+    void testInfer() {
         LogicalPlan plan = new LogicalPlanBuilder(scan1)
-                .join(scan2, JoinType.LEFT_OUTER_JOIN, Pair.of(0, 0))  // 
t1.id = t2.id
-                .filter(new GreaterThan(scan2.getOutput().get(0), 
Literal.of(1)))
+                .filter(new EqualTo(new Add(scan1.getOutput().get(0), 
scan1.getOutput().get(1)), Literal.of(1)))
                 .build();
 
         PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
-                .applyTopDown(new EliminateOuterJoin())
-                .matchesFromRoot(
-                        logicalFilter(
-                                logicalJoin().when(join -> 
join.getJoinType().isInnerJoin())
-                        )
+                .applyTopDown(new InferFilterNotNull())
+                .matches(
+                        // LogicalFilter ( predicates=((((id#0 + name#1) = 1) 
AND ( not id IS NULL)) AND ( not name IS NULL)) )
+                        logicalFilter().when(filter -> 
filter.getConjuncts().size() == 3)
                 );
     }
 
     @Test
-    void testEliminateRight() {
+    void testInferFail() {
         LogicalPlan plan = new LogicalPlanBuilder(scan1)
-                .join(scan2, JoinType.RIGHT_OUTER_JOIN, Pair.of(0, 0))  // 
t1.id = t2.id
-                .filter(new GreaterThan(scan1.getOutput().get(0), 
Literal.of(1)))
+                .filter(new IsNull(scan1.getOutput().get(0)))
                 .build();
 
         PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
-                .applyTopDown(new EliminateOuterJoin())
-                .matchesFromRoot(
-                        logicalFilter(
-                                logicalJoin().when(join -> 
join.getJoinType().isInnerJoin())
-                        )
+                .applyTopDown(new InferFilterNotNull())
+                .matches(
+                        // LogicalFilter ( predicates=id IS NULL )
+                        logicalFilter().when(filter -> 
filter.getConjuncts().size() == 1)
                 );
     }
 
     @Test
-    void testEliminateBoth() {
+    void testInferFailOr() {
         LogicalPlan plan = new LogicalPlanBuilder(scan1)
-                .join(scan2, JoinType.FULL_OUTER_JOIN, Pair.of(0, 0))  // 
t1.id = t2.id
-                .filter(new And(
-                        new GreaterThan(scan2.getOutput().get(0), 
Literal.of(1)),
-                        new GreaterThan(scan1.getOutput().get(0), 
Literal.of(1))))
+                .filter(new Or(new IsNull(scan1.getOutput().get(0)),
+                        new EqualTo(scan1.getOutput().get(0), Literal.of(1))))
                 .build();
 
         PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
-                .applyTopDown(new EliminateOuterJoin())
-                .matchesFromRoot(
-                        logicalFilter(
-                                logicalJoin().when(join -> 
join.getJoinType().isInnerJoin())
-                        )
+                .applyTopDown(new InferFilterNotNull())
+                .matches(
+                        // LogicalFilter ( predicates=(id IS NULL OR (id#0 = 
1)) )
+                        logicalFilter().when(filter -> 
filter.getConjuncts().size() == 1)
                 );
     }
 }
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
new file mode 100644
index 0000000000..596a08116d
--- /dev/null
+++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/sqltest/InferTest.java
@@ -0,0 +1,88 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.sqltest;
+
+import org.apache.doris.nereids.util.PlanChecker;
+
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+public class InferTest extends SqlTestBase {
+    @Test
+    void testInferNotNullAndInferPredicates() {
+        // Test InferNotNull, EliminateOuter, InferPredicate together
+        String sql = "select * from T1 left outer join T2 on T1.id = T2.id 
where T2.id = 4";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matchesFromRoot(
+                        innerLogicalJoin(
+                                logicalFilter().when(f -> 
f.getPredicate().toString().equals("(id#0 = 4)")),
+                                logicalFilter().when(f -> 
f.getPredicate().toString().equals("(id#2 = 4)"))
+                        )
+                );
+    }
+
+    @Test
+    void testInferNotNullFromFilterAndEliminateOuter2() {
+        String sql
+                = "select * from T1 right outer join T2 on T1.id = T2.id where 
T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .printlnTree()
+                .matchesFromRoot(
+                        innerLogicalJoin(
+                                logicalFilter().when(
+                                        f -> 
f.getPredicate().toString().equals("((id#0 = 4) OR (id#0 > 4))")),
+                                logicalOlapScan()
+                        )
+                );
+    }
+
+    @Test
+    void testInferNotNullFromFilterAndEliminateOuter3() {
+        String sql
+                = "select * from T1 full outer join T2 on T1.id = T2.id where 
T1.id = 4 OR (T1.id > 4 AND T2.score IS NULL)";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .rewrite()
+                .matchesFromRoot(
+                        logicalFilter(
+                                leftOuterLogicalJoin(
+                                        logicalFilter().when(
+                                                f -> 
f.getPredicate().toString().equals("((id#0 = 4) OR (id#0 > 4))")),
+                                        logicalOlapScan()
+                                )
+                        ).when(f -> f.getPredicate().toString()
+                                .equals("((id#0 = 4) OR ((id#0 > 4) AND score 
IS NULL))"))
+                );
+    }
+
+    @Test
+    @Disabled
+    void testInferNotNullFromJoinAndEliminateOuter() {
+        String sql
+                = "select * from (select T1.id from T1 left outer join T2 on 
T1.id = T2.id) T1 left semi join T3 on T1.id = T3.id";
+        PlanChecker.from(connectContext)
+                .analyze(sql)
+                .printlnTree()
+                .rewrite()
+                .printlnTree();
+    }
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]


Reply via email to