This is an automated email from the ASF dual-hosted git repository.

jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git


The following commit(s) were added to refs/heads/master by this push:
     new da8c53a831 [feat](Nereids): pushdown semijoin through agg. (#18105)
da8c53a831 is described below

commit da8c53a83190e342f1d2055b2f329ac10279b0d3
Author: jakevin <[email protected]>
AuthorDate: Mon Mar 27 15:27:44 2023 +0800

    [feat](Nereids): pushdown semijoin through agg. (#18105)
---
 .../doris/nereids/jobs/batch/NereidsRewriter.java  |  6 +-
 .../org/apache/doris/nereids/rules/RuleType.java   |  6 +-
 .../rewrite/logical/SemiJoinAggTranspose.java      | 64 ++++++++++++++++++++
 .../logical/SemiJoinAggTransposeProject.java       | 68 ++++++++++++++++++++++
 .../logical/SemiJoinAggTransposeProjectTest.java   | 56 ++++++++++++++++++
 .../rewrite/logical/SemiJoinAggTransposeTest.java  | 52 +++++++++++++++++
 6 files changed, 248 insertions(+), 4 deletions(-)

diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
index 26bfd2b5bb..ef830e4e9c 100644
--- 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
@@ -63,6 +63,8 @@ import 
org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet;
 import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin;
 import org.apache.doris.nereids.rules.rewrite.logical.PushdownLimit;
 import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
+import org.apache.doris.nereids.rules.rewrite.logical.SemiJoinAggTranspose;
+import 
org.apache.doris.nereids.rules.rewrite.logical.SemiJoinAggTransposeProject;
 import 
org.apache.doris.nereids.rules.rewrite.logical.SemiJoinLogicalJoinTranspose;
 import 
org.apache.doris.nereids.rules.rewrite.logical.SemiJoinLogicalJoinTransposeProject;
 import org.apache.doris.nereids.rules.rewrite.logical.SplitLimit;
@@ -161,7 +163,9 @@ public class NereidsRewriter extends BatchRewriteJob {
                 topDown(
                     // new SemiJoinCommute(),
                     new SemiJoinLogicalJoinTranspose(),
-                    new SemiJoinLogicalJoinTransposeProject()
+                    new SemiJoinLogicalJoinTransposeProject(),
+                    new SemiJoinAggTranspose(),
+                    new SemiJoinAggTransposeProject()
                 ),
 
                 topDown(
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index d08eb03aa4..c85fdfeef8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -134,6 +134,8 @@ public enum RuleType {
     LOGICAL_SEMI_JOIN_LOGICAL_JOIN_TRANSPOSE(RuleTypeClass.REWRITE),
     LOGICAL_SEMI_JOIN_LOGICAL_JOIN_TRANSPOSE_PROJECT(RuleTypeClass.REWRITE),
     LOGICAL_SEMI_JOIN_COMMUTE(RuleTypeClass.REWRITE),
+    LOGICAL_SEMI_JOIN_AGG_TRANSPOSE(RuleTypeClass.REWRITE),
+    LOGICAL_SEMI_JOIN_AGG_TRANSPOSE_PROJECT(RuleTypeClass.REWRITE),
 
     // expression of plan rewrite
     REWRITE_ONE_ROW_RELATION_EXPRESSION(RuleTypeClass.REWRITE),
@@ -189,7 +191,6 @@ public enum RuleType {
     COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE),
     INNER_TO_CROSS_JOIN(RuleTypeClass.REWRITE),
     CROSS_TO_INNER_JOIN(RuleTypeClass.REWRITE),
-    REWRITE_SENTINEL(RuleTypeClass.REWRITE),
 
     // split limit
     SPLIT_LIMIT(RuleTypeClass.REWRITE),
@@ -211,8 +212,7 @@ public enum RuleType {
     ADJUST_NULLABLE_ON_SORT(RuleTypeClass.REWRITE),
     ADJUST_NULLABLE_ON_TOP_N(RuleTypeClass.REWRITE),
 
-    // Join Reorder
-    JOIN_REORDER(RuleTypeClass.REWRITE),
+    REWRITE_SENTINEL(RuleTypeClass.REWRITE),
 
     // exploration rules
     TEST_EXPLORATION(RuleTypeClass.EXPLORATION),
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTranspose.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTranspose.java
new file mode 100644
index 0000000000..f6ecde1f4e
--- /dev/null
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTranspose.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Pushdown semi-join through agg
+ */
+public class SemiJoinAggTranspose extends OneRewriteRuleFactory {
+    @Override
+    public Rule build() {
+        return logicalJoin(logicalAggregate(), any())
+                .when(join -> join.getJoinType().isLeftSemiOrAntiJoin())
+                .then(join -> {
+                    LogicalAggregate<Plan> aggregate = join.left();
+                    Set<Slot> canPushDownSlots = new HashSet<>();
+                    if (aggregate.hasRepeat()) {
+                        // When there is a repeat, the push-down condition is 
consistent with the repeat
+                        
canPushDownSlots.addAll(aggregate.getSourceRepeat().get().getCommonGroupingSetExpressions());
+                    } else {
+                        for (Expression groupByExpression : 
aggregate.getGroupByExpressions()) {
+                            if (groupByExpression instanceof Slot) {
+                                canPushDownSlots.add((Slot) groupByExpression);
+                            }
+                        }
+                    }
+                    Set<Slot> leftOutputSet = join.left().getOutputSet();
+                    Set<Slot> conditionSlot = join.getConditionSlot()
+                            .stream()
+                            .filter(leftOutputSet::contains)
+                            .collect(Collectors.toSet());
+                    if (!canPushDownSlots.containsAll(conditionSlot)) {
+                        return null;
+                    }
+                    return 
aggregate.withChildren(join.withChildren(aggregate.child(), join.right()));
+                }).toRule(RuleType.LOGICAL_SEMI_JOIN_AGG_TRANSPOSE);
+    }
+}
diff --git 
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProject.java
 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProject.java
new file mode 100644
index 0000000000..1820cfde37
--- /dev/null
+++ 
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProject.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Pushdown semi-join through agg
+ */
+public class SemiJoinAggTransposeProject extends OneRewriteRuleFactory {
+    @Override
+    public Rule build() {
+        return logicalJoin(logicalProject(logicalAggregate()), any())
+                .when(join -> join.getJoinType().isLeftSemiOrAntiJoin())
+                .when(join -> join.left().getProjects().stream().allMatch(n -> 
n instanceof Slot))
+                .then(join -> {
+                    LogicalProject<LogicalAggregate<Plan>> project = 
join.left();
+                    LogicalAggregate<Plan> aggregate = project.child();
+                    Set<Slot> canPushDownSlots = new HashSet<>();
+                    if (aggregate.hasRepeat()) {
+                        // When there is a repeat, the push-down condition is 
consistent with the repeat
+                        
canPushDownSlots.addAll(aggregate.getSourceRepeat().get().getCommonGroupingSetExpressions());
+                    } else {
+                        for (Expression groupByExpression : 
aggregate.getGroupByExpressions()) {
+                            if (groupByExpression instanceof Slot) {
+                                canPushDownSlots.add((Slot) groupByExpression);
+                            }
+                        }
+                    }
+                    Set<Slot> leftOutputSet = join.left().getOutputSet();
+                    Set<Slot> conditionSlot = join.getConditionSlot()
+                            .stream()
+                            .filter(leftOutputSet::contains)
+                            .collect(Collectors.toSet());
+                    if (!canPushDownSlots.containsAll(conditionSlot)) {
+                        return null;
+                    }
+                    Plan newPlan = 
aggregate.withChildren(join.withChildren(aggregate.child(), join.right()));
+                    return project.withChildren(newPlan);
+                }).toRule(RuleType.LOGICAL_SEMI_JOIN_AGG_TRANSPOSE_PROJECT);
+    }
+}
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProjectTest.java
 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProjectTest.java
new file mode 100644
index 0000000000..d3cad3cae4
--- /dev/null
+++ 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProjectTest.java
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.LogicalPlanBuilder;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.nereids.util.PlanConstructor;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Test;
+
+class SemiJoinAggTransposeProjectTest implements MemoPatternMatchSupported {
+    private final LogicalOlapScan scan1 = 
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+    private final LogicalOlapScan scan2 = 
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
+
+    @Test
+    void simple() {
+        LogicalPlan plan = new LogicalPlanBuilder(scan1)
+                .aggAllUsingIndex(ImmutableList.of(0, 1), ImmutableList.of(0, 
1))
+                .project(ImmutableList.of(0))
+                .join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0))
+                .build();
+        PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+                .applyTopDown(new SemiJoinAggTransposeProject())
+                .printlnTree()
+                .matchesFromRoot(
+                        logicalProject(
+                                logicalAggregate(
+                                        leftSemiLogicalJoin()
+                                )
+                        )
+                );
+    }
+
+}
diff --git 
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeTest.java
 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeTest.java
new file mode 100644
index 0000000000..264b6fdc0d
--- /dev/null
+++ 
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeTest.java
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.LogicalPlanBuilder;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.nereids.util.PlanConstructor;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Test;
+
+class SemiJoinAggTransposeTest implements MemoPatternMatchSupported {
+    private final LogicalOlapScan scan1 = 
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+    private final LogicalOlapScan scan2 = 
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
+
+    @Test
+    void simple() {
+        LogicalPlan plan = new LogicalPlanBuilder(scan1)
+                .aggAllUsingIndex(ImmutableList.of(0, 1), ImmutableList.of(0, 
1))
+                .join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0))
+                .build();
+        PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+                .applyTopDown(new SemiJoinAggTranspose())
+                .matchesFromRoot(
+                        logicalAggregate(
+                                leftSemiLogicalJoin()
+                        )
+                );
+    }
+
+}


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to