This is an automated email from the ASF dual-hosted git repository.
jakevin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new da8c53a831 [feat](Nereids): pushdown semijoin through agg. (#18105)
da8c53a831 is described below
commit da8c53a83190e342f1d2055b2f329ac10279b0d3
Author: jakevin <[email protected]>
AuthorDate: Mon Mar 27 15:27:44 2023 +0800
[feat](Nereids): pushdown semijoin through agg. (#18105)
---
.../doris/nereids/jobs/batch/NereidsRewriter.java | 6 +-
.../org/apache/doris/nereids/rules/RuleType.java | 6 +-
.../rewrite/logical/SemiJoinAggTranspose.java | 64 ++++++++++++++++++++
.../logical/SemiJoinAggTransposeProject.java | 68 ++++++++++++++++++++++
.../logical/SemiJoinAggTransposeProjectTest.java | 56 ++++++++++++++++++
.../rewrite/logical/SemiJoinAggTransposeTest.java | 52 +++++++++++++++++
6 files changed, 248 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
index 26bfd2b5bb..ef830e4e9c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/batch/NereidsRewriter.java
@@ -63,6 +63,8 @@ import
org.apache.doris.nereids.rules.rewrite.logical.PruneOlapScanTablet;
import org.apache.doris.nereids.rules.rewrite.logical.PushFilterInsideJoin;
import org.apache.doris.nereids.rules.rewrite.logical.PushdownLimit;
import org.apache.doris.nereids.rules.rewrite.logical.ReorderJoin;
+import org.apache.doris.nereids.rules.rewrite.logical.SemiJoinAggTranspose;
+import
org.apache.doris.nereids.rules.rewrite.logical.SemiJoinAggTransposeProject;
import
org.apache.doris.nereids.rules.rewrite.logical.SemiJoinLogicalJoinTranspose;
import
org.apache.doris.nereids.rules.rewrite.logical.SemiJoinLogicalJoinTransposeProject;
import org.apache.doris.nereids.rules.rewrite.logical.SplitLimit;
@@ -161,7 +163,9 @@ public class NereidsRewriter extends BatchRewriteJob {
topDown(
// new SemiJoinCommute(),
new SemiJoinLogicalJoinTranspose(),
- new SemiJoinLogicalJoinTransposeProject()
+ new SemiJoinLogicalJoinTransposeProject(),
+ new SemiJoinAggTranspose(),
+ new SemiJoinAggTransposeProject()
),
topDown(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index d08eb03aa4..c85fdfeef8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -134,6 +134,8 @@ public enum RuleType {
LOGICAL_SEMI_JOIN_LOGICAL_JOIN_TRANSPOSE(RuleTypeClass.REWRITE),
LOGICAL_SEMI_JOIN_LOGICAL_JOIN_TRANSPOSE_PROJECT(RuleTypeClass.REWRITE),
LOGICAL_SEMI_JOIN_COMMUTE(RuleTypeClass.REWRITE),
+ LOGICAL_SEMI_JOIN_AGG_TRANSPOSE(RuleTypeClass.REWRITE),
+ LOGICAL_SEMI_JOIN_AGG_TRANSPOSE_PROJECT(RuleTypeClass.REWRITE),
// expression of plan rewrite
REWRITE_ONE_ROW_RELATION_EXPRESSION(RuleTypeClass.REWRITE),
@@ -189,7 +191,6 @@ public enum RuleType {
COUNT_DISTINCT_REWRITE(RuleTypeClass.REWRITE),
INNER_TO_CROSS_JOIN(RuleTypeClass.REWRITE),
CROSS_TO_INNER_JOIN(RuleTypeClass.REWRITE),
- REWRITE_SENTINEL(RuleTypeClass.REWRITE),
// split limit
SPLIT_LIMIT(RuleTypeClass.REWRITE),
@@ -211,8 +212,7 @@ public enum RuleType {
ADJUST_NULLABLE_ON_SORT(RuleTypeClass.REWRITE),
ADJUST_NULLABLE_ON_TOP_N(RuleTypeClass.REWRITE),
- // Join Reorder
- JOIN_REORDER(RuleTypeClass.REWRITE),
+ REWRITE_SENTINEL(RuleTypeClass.REWRITE),
// exploration rules
TEST_EXPLORATION(RuleTypeClass.EXPLORATION),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTranspose.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTranspose.java
new file mode 100644
index 0000000000..f6ecde1f4e
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTranspose.java
@@ -0,0 +1,64 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Pushdown semi-join through agg
+ */
+public class SemiJoinAggTranspose extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalJoin(logicalAggregate(), any())
+ .when(join -> join.getJoinType().isLeftSemiOrAntiJoin())
+ .then(join -> {
+ LogicalAggregate<Plan> aggregate = join.left();
+ Set<Slot> canPushDownSlots = new HashSet<>();
+ if (aggregate.hasRepeat()) {
+ // When there is a repeat, the push-down condition is
consistent with the repeat
+
canPushDownSlots.addAll(aggregate.getSourceRepeat().get().getCommonGroupingSetExpressions());
+ } else {
+ for (Expression groupByExpression :
aggregate.getGroupByExpressions()) {
+ if (groupByExpression instanceof Slot) {
+ canPushDownSlots.add((Slot) groupByExpression);
+ }
+ }
+ }
+ Set<Slot> leftOutputSet = join.left().getOutputSet();
+ Set<Slot> conditionSlot = join.getConditionSlot()
+ .stream()
+ .filter(leftOutputSet::contains)
+ .collect(Collectors.toSet());
+ if (!canPushDownSlots.containsAll(conditionSlot)) {
+ return null;
+ }
+ return
aggregate.withChildren(join.withChildren(aggregate.child(), join.right()));
+ }).toRule(RuleType.LOGICAL_SEMI_JOIN_AGG_TRANSPOSE);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProject.java
new file mode 100644
index 0000000000..1820cfde37
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProject.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import java.util.HashSet;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+/**
+ * Pushdown semi-join through agg
+ */
+public class SemiJoinAggTransposeProject extends OneRewriteRuleFactory {
+ @Override
+ public Rule build() {
+ return logicalJoin(logicalProject(logicalAggregate()), any())
+ .when(join -> join.getJoinType().isLeftSemiOrAntiJoin())
+ .when(join -> join.left().getProjects().stream().allMatch(n ->
n instanceof Slot))
+ .then(join -> {
+ LogicalProject<LogicalAggregate<Plan>> project =
join.left();
+ LogicalAggregate<Plan> aggregate = project.child();
+ Set<Slot> canPushDownSlots = new HashSet<>();
+ if (aggregate.hasRepeat()) {
+ // When there is a repeat, the push-down condition is
consistent with the repeat
+
canPushDownSlots.addAll(aggregate.getSourceRepeat().get().getCommonGroupingSetExpressions());
+ } else {
+ for (Expression groupByExpression :
aggregate.getGroupByExpressions()) {
+ if (groupByExpression instanceof Slot) {
+ canPushDownSlots.add((Slot) groupByExpression);
+ }
+ }
+ }
+ Set<Slot> leftOutputSet = join.left().getOutputSet();
+ Set<Slot> conditionSlot = join.getConditionSlot()
+ .stream()
+ .filter(leftOutputSet::contains)
+ .collect(Collectors.toSet());
+ if (!canPushDownSlots.containsAll(conditionSlot)) {
+ return null;
+ }
+ Plan newPlan =
aggregate.withChildren(join.withChildren(aggregate.child(), join.right()));
+ return project.withChildren(newPlan);
+ }).toRule(RuleType.LOGICAL_SEMI_JOIN_AGG_TRANSPOSE_PROJECT);
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProjectTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProjectTest.java
new file mode 100644
index 0000000000..d3cad3cae4
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeProjectTest.java
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.LogicalPlanBuilder;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.nereids.util.PlanConstructor;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Test;
+
+class SemiJoinAggTransposeProjectTest implements MemoPatternMatchSupported {
+ private final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ private final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
+
+ @Test
+ void simple() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .aggAllUsingIndex(ImmutableList.of(0, 1), ImmutableList.of(0,
1))
+ .project(ImmutableList.of(0))
+ .join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new SemiJoinAggTransposeProject())
+ .printlnTree()
+ .matchesFromRoot(
+ logicalProject(
+ logicalAggregate(
+ leftSemiLogicalJoin()
+ )
+ )
+ );
+ }
+
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeTest.java
new file mode 100644
index 0000000000..264b6fdc0d
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/logical/SemiJoinAggTransposeTest.java
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite.logical;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.trees.plans.JoinType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.LogicalPlanBuilder;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.MemoTestUtils;
+import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.nereids.util.PlanConstructor;
+
+import com.google.common.collect.ImmutableList;
+import org.junit.jupiter.api.Test;
+
+class SemiJoinAggTransposeTest implements MemoPatternMatchSupported {
+ private final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ private final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
+
+ @Test
+ void simple() {
+ LogicalPlan plan = new LogicalPlanBuilder(scan1)
+ .aggAllUsingIndex(ImmutableList.of(0, 1), ImmutableList.of(0,
1))
+ .join(scan2, JoinType.LEFT_SEMI_JOIN, Pair.of(0, 0))
+ .build();
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .applyTopDown(new SemiJoinAggTranspose())
+ .matchesFromRoot(
+ logicalAggregate(
+ leftSemiLogicalJoin()
+ )
+ );
+ }
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]