This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 47019133c0e [improvement](Nereids) Support to remove sort which is
under table sink (#31751) (#32337)
47019133c0e is described below
commit 47019133c0e4d6065275a1ae01eaa1e314578958
Author: seawinde <149132972+seawi...@users.noreply.github.com>
AuthorDate: Sun Mar 17 15:45:53 2024 +0800
[improvement](Nereids) Support to remove sort which is under table sink
(#31751) (#32337)
---
.../doris/nereids/rules/rewrite/EliminateSort.java | 17 ++--
.../trees/plans/logical/LogicalOlapTableSink.java | 2 +-
.../trees/plans/logical/LogicalTableSink.java | 43 ++++++++
.../plans/physical/PhysicalOlapTableSink.java | 2 +-
.../trees/plans/physical/PhysicalTableSink.java | 45 +++++++++
.../nereids/trees/plans/visitor/SinkVisitor.java | 10 ++
.../nereids/rules/rewrite/EliminateSortTest.java | 110 ++++++++++++++++++++-
7 files changed, 219 insertions(+), 10 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSort.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSort.java
index 50fe0bc414f..9dd80771785 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSort.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/EliminateSort.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalSort;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTableSink;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
@@ -29,10 +30,10 @@ import java.util.ArrayList;
import java.util.List;
/**
- * Eliminate sort that is not directly below result sink
+ * Eliminate sort that is not directly below result sink, if there is project
between result sink and sort,
+ * the sort will not be eliminated.
* Note we have put limit in sort node so that we don't need to consider limit
*/
-
public class EliminateSort extends DefaultPlanRewriter<Boolean> implements
CustomRewriter {
@Override
public Plan rewriteRoot(Plan plan, JobContext jobContext) {
@@ -45,6 +46,7 @@ public class EliminateSort extends
DefaultPlanRewriter<Boolean> implements Custo
List<Plan> newChildren = new ArrayList<>();
boolean hasNewChildren = false;
for (Plan child : plan.children()) {
+ // eliminate sort default
Plan newChild = child.accept(this, true);
if (newChild != child) {
hasNewChildren = true;
@@ -64,14 +66,17 @@ public class EliminateSort extends
DefaultPlanRewriter<Boolean> implements Custo
@Override
public Plan visitLogicalProject(LogicalProject<? extends Plan> project,
Boolean eliminateSort) {
+ // sometimes there is project between logicalResultSink and sort,
should skip eliminate
return skipEliminateSort(project, eliminateSort);
}
@Override
- public Plan visitLogicalSink(LogicalSink<? extends Plan> sink, Boolean
eliminateSort) {
- // 1. table sink: eliminate -> true
- // 2. sink -> tablesink -> olaptablesink
- return skipEliminateSort(sink, eliminateSort);
+ public Plan visitLogicalSink(LogicalSink<? extends Plan> logicalSink,
Boolean eliminateSort) {
+ if (logicalSink instanceof LogicalTableSink) {
+ // eliminate sort
+ return visit(logicalSink, true);
+ }
+ return skipEliminateSort(logicalSink, eliminateSort);
}
private Plan skipEliminateSort(Plan plan, Boolean eliminateSort) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java
index 1a298ea8c40..c5e3336dabb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapTableSink.java
@@ -41,7 +41,7 @@ import java.util.Optional;
/**
* logical olap table sink for insert command
*/
-public class LogicalOlapTableSink<CHILD_TYPE extends Plan> extends
LogicalSink<CHILD_TYPE>
+public class LogicalOlapTableSink<CHILD_TYPE extends Plan> extends
LogicalTableSink<CHILD_TYPE>
implements Sink, PropagateFuncDeps {
// bound data sink
private final Database database;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTableSink.java
new file mode 100644
index 00000000000..90133274f6e
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalTableSink.java
@@ -0,0 +1,43 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.logical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Logical table sink for all table type sink
+ */
+public abstract class LogicalTableSink<CHILD_TYPE extends Plan> extends
LogicalSink<CHILD_TYPE> {
+ public LogicalTableSink(PlanType type,
+ List<NamedExpression> outputExprs, CHILD_TYPE child) {
+ super(type, outputExprs, child);
+ }
+
+ public LogicalTableSink(PlanType type, List<NamedExpression> outputExprs,
+ Optional<GroupExpression> groupExpression,
+ Optional<LogicalProperties> logicalProperties, CHILD_TYPE child) {
+ super(type, outputExprs, groupExpression, logicalProperties, child);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java
index 86a6a4b8c61..debc0ece6c2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapTableSink.java
@@ -50,7 +50,7 @@ import java.util.Set;
/**
* physical olap table sink for insert command
*/
-public class PhysicalOlapTableSink<CHILD_TYPE extends Plan> extends
PhysicalSink<CHILD_TYPE> implements Sink {
+public class PhysicalOlapTableSink<CHILD_TYPE extends Plan> extends
PhysicalTableSink<CHILD_TYPE> implements Sink {
private final Database database;
private final OlapTable targetTable;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTableSink.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTableSink.java
new file mode 100644
index 00000000000..5932c4f0999
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalTableSink.java
@@ -0,0 +1,45 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans.physical;
+
+import org.apache.doris.nereids.memo.GroupExpression;
+import org.apache.doris.nereids.properties.LogicalProperties;
+import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.PlanType;
+import org.apache.doris.statistics.Statistics;
+
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+import java.util.Optional;
+
+/**
+ * Physical table sink for all table type sink
+ */
+public abstract class PhysicalTableSink<CHILD_TYPE extends Plan> extends
PhysicalSink<CHILD_TYPE> {
+
+ public PhysicalTableSink(PlanType type, List<NamedExpression> outputExprs,
+ Optional<GroupExpression> groupExpression,
+ LogicalProperties logicalProperties,
+ @Nullable PhysicalProperties physicalProperties,
+ Statistics statistics, CHILD_TYPE child) {
+ super(type, outputExprs, groupExpression, logicalProperties,
physicalProperties, statistics, child);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java
index 343b3a4cf01..fcb0b474e47 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/SinkVisitor.java
@@ -25,11 +25,13 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalFileSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink;
import org.apache.doris.nereids.trees.plans.logical.LogicalSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalTableSink;
import
org.apache.doris.nereids.trees.plans.physical.PhysicalDeferMaterializeResultSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalFileSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapTableSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalResultSink;
import org.apache.doris.nereids.trees.plans.physical.PhysicalSink;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalTableSink;
/**
* sink visitor
@@ -64,6 +66,10 @@ public interface SinkVisitor<R, C> {
return visitLogicalSink(fileSink, context);
}
+ default R visitLogicalTableSink(LogicalTableSink<? extends Plan>
logicalTableSink, C context) {
+ return visitLogicalSink(logicalTableSink, context);
+ }
+
default R visitLogicalOlapTableSink(LogicalOlapTableSink<? extends Plan>
olapTableSink, C context) {
return visitLogicalSink(olapTableSink, context);
}
@@ -85,6 +91,10 @@ public interface SinkVisitor<R, C> {
return visitPhysicalSink(fileSink, context);
}
+ default R visitPhysicalTableSink(PhysicalTableSink<? extends Plan>
physicalTableSink, C context) {
+ return visitPhysicalSink(physicalTableSink, context);
+ }
+
default R visitPhysicalOlapTableSink(PhysicalOlapTableSink<? extends Plan>
olapTableSink, C context) {
return visitPhysicalSink(olapTableSink, context);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java
index 8dd2eb5f798..5df92782527 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/EliminateSortTest.java
@@ -17,12 +17,25 @@
package org.apache.doris.nereids.rules.rewrite;
+import org.apache.doris.catalog.Database;
+import org.apache.doris.nereids.properties.OrderKey;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink;
+import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
+import org.apache.doris.nereids.util.LogicalPlanBuilder;
import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.MemoTestUtils;
import org.apache.doris.nereids.util.PlanChecker;
+import org.apache.doris.nereids.util.PlanConstructor;
import org.apache.doris.utframe.TestWithFeService;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
+import java.util.stream.Collectors;
+
/**
* column prune ut.
*/
@@ -37,15 +50,108 @@ class EliminateSortTest extends TestWithFeService
implements MemoPatternMatchSup
}
@Test
- void test() {
+ void testEliminateSortWhenCTE() {
+ PlanChecker.from(connectContext)
+ .analyze("with cte_test as (\n"
+ + "select id, name, age from student order by id\n"
+ + ")\n"
+ + "select * from cte_test c1\n"
+ + "join student\n"
+ + "join cte_test c2\n"
+ + "where c1.id = student.id")
+ .rewrite()
+ .nonMatch(logicalSort());
+
PlanChecker.from(connectContext)
- .analyze("select * from student order by id")
+ .analyze("with cte_test as (\n"
+ + "select id, name, age from student\n"
+ + ")\n"
+ + "select * from cte_test c1\n"
+ + "join student\n"
+ + "join cte_test c2\n"
+ + "where c1.id = student.id order by c1.age")
.rewrite()
.matches(logicalSort());
+
+ PlanChecker.from(connectContext)
+ .analyze("select t.age from\n"
+ + "(\n"
+ + "with cte_test as (\n"
+ + "select id, name, age from student order by id\n"
+ + ")\n"
+ + "select c1.id, student.name, c2.age from cte_test
c1\n"
+ + "join student\n"
+ + "join cte_test c2\n"
+ + "where c1.id = student.id order by c1.age\n"
+ + ") t order by t.id")
+ .rewrite()
+ .matches(logicalSort());
+ }
+
+ @Test
+ void testEliminateSortUnderTableSink() {
+ // topN under table sink should not be removed
+ LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0);
+ LogicalPlan plan = new LogicalPlanBuilder(scan)
+ .sort(scan.getOutput().stream().map(c -> new OrderKey(c, true,
true)).collect(Collectors.toList()))
+ .limit(1, 1).build();
+ plan = new LogicalOlapTableSink<>(new Database(), scan.getTable(),
scan.getTable().getBaseSchema(),
+ new ArrayList<>(),
plan.getOutput().stream().map(NamedExpression.class::cast).collect(
+ Collectors.toList()), false, DMLCommandType.NONE, plan);
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .rewrite()
+ .nonMatch(logicalSort())
+ .matches(logicalTopN());
+
+ // sort under table sink should be removed
+ scan = PlanConstructor.newLogicalOlapScan(0, "t2", 0);
+ plan = new LogicalPlanBuilder(scan)
+ .sort(scan.getOutput().stream().map(c -> new OrderKey(c, true,
true)).collect(Collectors.toList()))
+ .build();
+ plan = new LogicalOlapTableSink<>(new Database(), scan.getTable(),
scan.getTable().getBaseSchema(),
+ new ArrayList<>(),
plan.getOutput().stream().map(NamedExpression.class::cast).collect(
+ Collectors.toList()), false, DMLCommandType.NONE, plan);
+ PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
+ .rewrite()
+ .nonMatch(logicalSort());
+ }
+
+ @Test
+ void testEliminateSortInUnion() {
+ PlanChecker.from(connectContext)
+ .analyze("SELECT * FROM (SELECT * FROM student UNION SELECT *
FROM student ORDER BY id) u LIMIT 1")
+ .rewrite()
+ .nonMatch(logicalSort());
+ }
+
+ @Test
+ void testEliminateSortInSubquery() {
PlanChecker.from(connectContext)
.analyze("select count(*) from (select * from student order by
id) t")
.rewrite()
.nonMatch(logicalSort());
+ PlanChecker.from(connectContext)
+ .analyze("select \n"
+ + " id, \n"
+ + " name \n"
+ + "from \n"
+ + " (\n"
+ + " select \n"
+ + " id, \n"
+ + " name, \n"
+ + " age \n"
+ + " from \n"
+ + " (\n"
+ + " select \n"
+ + " * \n"
+ + " from \n"
+ + " student\n"
+ + " ) s \n"
+ + " order by \n"
+ + " id\n"
+ + " ) t")
+ .rewrite()
+ .nonMatch(logicalSort());
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
