This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch nested_column_prune
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/nested_column_prune by this
push:
new 70cb7f63446 push down project through join/union
70cb7f63446 is described below
commit 70cb7f63446715fba86989229e9b67517e8632bc
Author: 924060929 <[email protected]>
AuthorDate: Wed Oct 22 20:55:11 2025 +0800
push down project through join/union
---
.idea/vcs.xml | 52 ++--
.../org/apache/doris/nereids/rules/RuleSet.java | 2 +
.../org/apache/doris/nereids/rules/RuleType.java | 3 +
.../rewrite/AccessPathExpressionCollector.java | 7 +
.../rules/rewrite/PushDownNestedColumn.java | 299 +++++++++++++++++++++
.../trees/expressions/PreferPushDownProject.java | 41 +++
.../expressions/functions/scalar/ArrayMap.java | 3 +-
.../expressions/functions/scalar/ElementAt.java | 3 +-
.../functions/scalar/HighOrderFunction.java | 3 +-
.../functions/scalar/MapContainsEntry.java | 3 +-
.../functions/scalar/MapContainsKey.java | 3 +-
.../functions/scalar/MapContainsValue.java | 3 +-
.../expressions/functions/scalar/MapEntries.java | 3 +-
.../expressions/functions/scalar/MapKeys.java | 3 +-
.../expressions/functions/scalar/MapValues.java | 3 +-
.../functions/scalar/StructElement.java | 3 +-
.../rules/rewrite/PruneNestedColumnTest.java | 59 +++-
17 files changed, 450 insertions(+), 43 deletions(-)
diff --git a/.idea/vcs.xml b/.idea/vcs.xml
index 0a185d9bdc7..8b083db2f94 100644
--- a/.idea/vcs.xml
+++ b/.idea/vcs.xml
@@ -1,34 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
-<!--
- Licensed to the Apache Software Foundation (ASF) under one or more
- contributor license agreements. See the NOTICE file distributed with
- this work for additional information regarding copyright ownership.
- The ASF licenses this file to You under the Apache License, Version 2.0
- (the "License"); you may not use this file except in compliance with
- the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
- -->
<project version="4">
- <component name="IssueNavigationConfiguration">
- <option name="links">
- <list>
- <IssueNavigationLink>
- <option name="issueRegexp" value="#(\d+)" />
- <option name="linkRegexp"
value="https://github.com/apache/doris/pull/$1"; />
- </IssueNavigationLink>
- </list>
- </option>
- </component>
- <component name="VcsDirectoryMappings">
- <mapping directory="$PROJECT_DIR$" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
- <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
- </component>
-</project>
+ <component name="IssueNavigationConfiguration">
+ <option name="links">
+ <list>
+ <IssueNavigationLink>
+ <option name="issueRegexp" value="#(\d+)" />
+ <option name="linkRegexp"
value="https://github.com/apache/doris/pull/$1"; />
+ </IssueNavigationLink>
+ </list>
+ </option>
+ </component>
+ <component name="VcsDirectoryMappings">
+ <mapping directory="$PROJECT_DIR$" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/be/src/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/apache-orc" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/clucene" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/faiss" vcs="Git" />
+ <mapping directory="$PROJECT_DIR$/contrib/openblas" vcs="Git" />
+ </component>
+</project>
\ No newline at end of file
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 6e47834d6c9..f78f0bec970 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -114,6 +114,7 @@ import
org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughSort;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughWindow;
import org.apache.doris.nereids.rules.rewrite.PushDownJoinOtherCondition;
import org.apache.doris.nereids.rules.rewrite.PushDownLimitDistinctThroughJoin;
+import org.apache.doris.nereids.rules.rewrite.PushDownNestedColumn;
import org.apache.doris.nereids.rules.rewrite.PushDownProjectThroughLimit;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughJoin;
import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin;
@@ -165,6 +166,7 @@ public class RuleSet {
new PushDownFilterThroughSetOperation(),
new PushDownFilterThroughGenerate(),
new PushDownProjectThroughLimit(),
+ new PushDownNestedColumn(),
new EliminateOuterJoin(),
new MergeProjectable(),
new MergeFilters(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 163c334290a..efdcc150b7b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -194,6 +194,9 @@ public enum RuleType {
PUSH_DOWN_PREDICATE_THROUGH_AGGREGATION(RuleTypeClass.REWRITE),
PUSH_DOWN_PREDICATE_THROUGH_REPEAT(RuleTypeClass.REWRITE),
PUSH_DOWN_EXPRESSIONS_IN_HASH_CONDITIONS(RuleTypeClass.REWRITE),
+ // Pushdown nested column project
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN(RuleTypeClass.REWRITE),
+ PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION(RuleTypeClass.REWRITE),
// Pushdown filter
PUSH_DOWN_FILTER_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSH_DOWN_FILTER_THROUGH_LEFT_SEMI_JOIN(RuleTypeClass.REWRITE),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
index 7caa3e59c45..8bd794f5422 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AccessPathExpressionCollector.java
@@ -40,6 +40,7 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySortBy;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ArraySplit;
import org.apache.doris.nereids.trees.expressions.functions.scalar.ElementAt;
import org.apache.doris.nereids.trees.expressions.functions.scalar.Lambda;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsEntry;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsKey;
import
org.apache.doris.nereids.trees.expressions.functions.scalar.MapContainsValue;
import org.apache.doris.nereids.trees.expressions.functions.scalar.MapKeys;
@@ -198,6 +199,12 @@ public class AccessPathExpressionCollector extends
DefaultExpressionVisitor<Void
return continueCollectAccessPath(mapContainsValue.getArgument(0),
context);
}
+ @Override
+ public Void visitMapContainsEntry(MapContainsEntry mapContainsEntry,
CollectorContext context) {
+ context.accessPathBuilder.addPrefix("*");
+ return continueCollectAccessPath(mapContainsEntry.getArgument(0),
context);
+ }
+
@Override
public Void visitArrayMap(ArrayMap arrayMap, CollectorContext context) {
// ARRAY_MAP(lambda, <arr> [ , <arr> ... ] )
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
new file mode 100644
index 00000000000..a6e790820a5
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownNestedColumn.java
@@ -0,0 +1,299 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.rewrite;
+
+import org.apache.doris.common.Pair;
+import org.apache.doris.nereids.StatementContext;
+import org.apache.doris.nereids.pattern.MatchingContext;
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.SetOperation.Qualifier;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalOneRowRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.trees.plans.logical.LogicalUnion;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.ImmutableCollection;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Multimap;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.function.Function;
+
+/** PushDownNestedColumnThroughJoin */
+public class PushDownNestedColumn implements RewriteRuleFactory {
+ @Override
+ public List<Rule> buildRules() {
+ return ImmutableList.of(
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_JOIN.build(
+ logicalProject(logicalJoin()).thenApply(this::pushThroughJoin)
+ ),
+ RuleType.PUSH_DOWN_NESTED_COLUMN_THROUGH_UNION.build(
+ logicalProject(
+ logicalUnion().when(u -> u.getQualifier() ==
Qualifier.ALL)
+ ).thenApply(this::pushThroughUnion)
+ )
+ );
+ }
+
+ private Plan
pushThroughJoin(MatchingContext<LogicalProject<LogicalJoin<Plan, Plan>>> ctx) {
+ LogicalProject<LogicalJoin<Plan, Plan>> project = ctx.root;
+ LogicalJoin<Plan, Plan> join = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, join);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ =
pushdownProjectHelper.pushDownExpressions(project.getProjects());
+
+ if (pushProjects.first) {
+ List<Plan> newJoinChildren =
pushdownProjectHelper.buildNewChildren();
+ return new LogicalProject<>(
+ pushProjects.second,
+ join.withChildren(newJoinChildren)
+ );
+ }
+ return project;
+ }
+
+ private Plan
pushThroughUnion(MatchingContext<LogicalProject<LogicalUnion>> ctx) {
+ LogicalProject<LogicalUnion> project = ctx.root;
+ LogicalUnion union = project.child();
+ PushdownProjectHelper pushdownProjectHelper
+ = new PushdownProjectHelper(ctx.statementContext, project);
+
+ Pair<Boolean, List<NamedExpression>> pushProjects
+ =
pushdownProjectHelper.pushDownExpressions(project.getProjects());
+ if (pushProjects.first) {
+ List<NamedExpression> unionOutputs = union.getOutputs();
+ Map<Slot, Integer> slotToColumnIndex = new LinkedHashMap<>();
+ for (int i = 0; i < unionOutputs.size(); i++) {
+ NamedExpression output = unionOutputs.get(i);
+ slotToColumnIndex.put(output.toSlot(), i);
+ }
+
+ Collection<NamedExpression> pushDownProjections
+ = pushdownProjectHelper.childToPushDownProjects.values();
+ List<Plan> newChildren = new ArrayList<>();
+ List<List<SlotReference>> newChildrenOutputs = new ArrayList<>();
+ for (Plan child : union.children()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex =
slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return
child.getOutput().get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ child
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ for (List<NamedExpression> originConstantExprs :
union.getConstantExprsList()) {
+ List<NamedExpression> pushedOutput = replaceSlot(
+ ctx.statementContext,
+ pushDownProjections,
+ slot -> {
+ Integer sourceColumnIndex =
slotToColumnIndex.get(slot);
+ if (sourceColumnIndex != null) {
+ return
originConstantExprs.get(sourceColumnIndex).toSlot();
+ }
+ return slot;
+ }
+ );
+
+ LogicalOneRowRelation originOneRowRelation = new
LogicalOneRowRelation(
+ ctx.statementContext.getNextRelationId(),
+ originConstantExprs
+ );
+
+ LogicalProject<Plan> newChild = new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(originOneRowRelation.getOutput())
+ .addAll(pushedOutput)
+ .build(),
+ originOneRowRelation
+ );
+
+ newChildrenOutputs.add((List) newChild.getOutput());
+ newChildren.add(newChild);
+ }
+
+ List<NamedExpression> newUnionOutputs = new
ArrayList<>(union.getOutputs());
+ for (NamedExpression projection : pushDownProjections) {
+ newUnionOutputs.add(projection.toSlot());
+ }
+
+ return new LogicalProject<>(
+ pushProjects.second,
+ new LogicalUnion(
+ union.getQualifier(),
+ newUnionOutputs,
+ newChildrenOutputs,
+ ImmutableList.of(),
+ union.hasPushedFilter(),
+ newChildren
+ )
+ );
+ }
+ return project;
+ }
+
+ private List<NamedExpression> replaceSlot(
+ StatementContext statementContext,
+ Collection<NamedExpression> pushDownProjections,
+ Function<Slot, Slot> slotReplace) {
+ List<NamedExpression> pushedOutput = new ArrayList<>();
+ for (NamedExpression projection : pushDownProjections) {
+ NamedExpression newOutput = (NamedExpression)
projection.rewriteUp(e -> {
+ if (e instanceof Slot) {
+ Slot newSlot = slotReplace.apply((Slot) e);
+ if (newSlot != null) {
+ return newSlot;
+ }
+ }
+ return e;
+ });
+ if (newOutput instanceof Alias) {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(),
newOutput.child(0)));
+ } else {
+ pushedOutput.add(new Alias(statementContext.getNextExprId(),
newOutput));
+ }
+ }
+ return pushedOutput;
+ }
+
+ private static class PushdownProjectHelper {
+ private final Plan plan;
+ private final StatementContext statementContext;
+ private final Map<Expression, Pair<Slot, Plan>> exprToChildAndSlot;
+ private final Multimap<Plan, NamedExpression> childToPushDownProjects;
+
+ public PushdownProjectHelper(StatementContext statementContext, Plan
plan) {
+ this.statementContext = statementContext;
+ this.plan = plan;
+ this.exprToChildAndSlot = new LinkedHashMap<>();
+ this.childToPushDownProjects = ArrayListMultimap.create();
+ }
+
+ public <C extends Collection<E>, E extends Expression> Pair<Boolean,
C> pushDownExpressions(C expressions) {
+ ImmutableCollection.Builder<E> builder;
+ if (expressions instanceof List) {
+ builder =
ImmutableList.builderWithExpectedSize(expressions.size());
+ } else {
+ builder =
ImmutableSet.builderWithExpectedSize(expressions.size());
+ }
+
+ boolean extracted = false;
+ for (E expression : expressions) {
+ Optional<E> result = pushDownExpression(expression);
+ if (!result.isPresent()) {
+ builder.add(expression);
+ } else {
+ extracted = true;
+ builder.add(result.get());
+ }
+ }
+
+ if (extracted) {
+ return Pair.of(true, (C) builder.build());
+ } else {
+ return Pair.of(false, expressions);
+ }
+ }
+
+ public <E extends Expression> Optional<E> pushDownExpression(E
expression) {
+ if (!(expression instanceof PreferPushDownProject
+ || (expression instanceof Alias && expression.child(0)
instanceof PreferPushDownProject))) {
+ return Optional.empty();
+ }
+ Pair<Slot, Plan> existPushdown =
exprToChildAndSlot.get(expression);
+ if (existPushdown != null) {
+ return Optional.of((E) existPushdown.first);
+ }
+
+ Alias pushDownAlias = null;
+ if (expression instanceof Alias) {
+ pushDownAlias = (Alias) expression;
+ } else {
+ pushDownAlias = new Alias(statementContext.getNextExprId(),
expression);
+ }
+
+ Set<Slot> inputSlots = expression.getInputSlots();
+ for (Plan child : plan.children()) {
+ if (child.getOutputSet().containsAll(inputSlots)) {
+ Slot remaimSlot = pushDownAlias.toSlot();
+ exprToChildAndSlot.put(expression, Pair.of(remaimSlot,
child));
+ childToPushDownProjects.put(child, pushDownAlias);
+ return Optional.of((E) remaimSlot);
+ }
+ }
+ return Optional.empty();
+ }
+
+ public List<Plan> buildNewChildren() {
+ if (childToPushDownProjects.isEmpty()) {
+ return plan.children();
+ }
+ ImmutableList.Builder<Plan> newChildren =
ImmutableList.builderWithExpectedSize(plan.arity());
+ for (Plan child : plan.children()) {
+ Collection<NamedExpression> newProject =
childToPushDownProjects.get(child);
+ if (newProject.isEmpty()) {
+ newChildren.add(child);
+ } else {
+ newChildren.add(
+ new LogicalProject<>(
+ ImmutableList.<NamedExpression>builder()
+ .addAll(child.getOutput())
+ .addAll(newProject)
+ .build(),
+ child
+ )
+ );
+ }
+ }
+ return newChildren.build();
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
new file mode 100644
index 00000000000..4c9c3bebe5a
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/PreferPushDownProject.java
@@ -0,0 +1,41 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.expressions;
+
+import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait;
+
+/**
+ * some expressions prefer push down project under the plan,
+ *
+ * e.g. the element_at function prefer push down to a project in under the
plan:
+ * Project(element_at(left.column, 'id') as a)
+ * |
+ * Join
+ * / |
+ * left right
+ *
+ * we will push down project through the Join, so we can prune the column
which has complex type:
+ *
+ * Join
+ * / \
+ * Project(element_at(left.column, 'id') as a, ...) right
+ * |
+ * left
+ */
+public interface PreferPushDownProject extends ExpressionTrait {
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
index 5d74f5e1457..1a1a246cea6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ArrayMap.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.ArrayItemReference;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import
org.apache.doris.nereids.trees.expressions.functions.ImplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'array_map'.
*/
public class ArrayMap extends ScalarFunction
- implements ImplicitlyCastableSignature, PropagateNullable {
+ implements ImplicitlyCastableSignature, PropagateNullable,
PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new
FollowToAnyDataType(0)).args(LambdaType.INSTANCE)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
index 99f63bab15c..10d7cfe1d94 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/ElementAt.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
@@ -40,7 +41,7 @@ import java.util.List;
* ScalarFunction 'element_at'. This class is generated by GenerateFunction.
*/
public class ElementAt extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNullable {
+ implements BinaryExpression, ExplicitlyCastableSignature,
AlwaysNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(new FollowToAnyDataType(0))
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
index a929f2dd5d5..e7a054fc864 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/HighOrderFunction.java
@@ -18,6 +18,7 @@
package org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.types.LambdaType;
import org.apache.doris.nereids.types.coercion.AnyDataType;
@@ -29,7 +30,7 @@ import java.util.List;
/**
* Interface of HighOrderFunction, provide default FunctionSignature of
LambdaType argument
*/
-public interface HighOrderFunction extends ExplicitlyCastableSignature {
+public interface HighOrderFunction extends ExplicitlyCastableSignature,
PreferPushDownProject {
@Override
default List<FunctionSignature> getSignatures() {
return new ImmutableList.Builder<FunctionSignature>()
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
index b8ecc8667c7..517bfaa7748 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsEntry.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.TernaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_entry'.
*/
public class MapContainsEntry extends ScalarFunction
- implements TernaryExpression, ExplicitlyCastableSignature {
+ implements TernaryExpression, ExplicitlyCastableSignature,
PreferPushDownProject {
/**
* Basic signature, forcing search key/value type to convert to the same
type as map's key/value type.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
index 4fc4c32f233..845e84f71b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsKey.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_key'. This class is generated by
GenerateFunction.
*/
public class MapContainsKey extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature,
PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
index a60f6653cde..fd42c42fb65 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapContainsValue.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression;
import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor;
@@ -36,7 +37,7 @@ import java.util.List;
* ScalarFunction 'map_contains_value'. This class is generated by
GenerateFunction.
*/
public class MapContainsValue extends ScalarFunction
- implements BinaryExpression, ExplicitlyCastableSignature {
+ implements BinaryExpression, ExplicitlyCastableSignature,
PreferPushDownProject {
public static final List<FunctionSignature> FOLLOW_DATATYPE_SIGNATURE =
ImmutableList.of(
FunctionSignature.ret(BooleanType.INSTANCE)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
index 4be48638e5e..22a14c31df3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapEntries.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.CustomSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.functions.SearchSignature;
@@ -41,7 +42,7 @@ import java.util.List;
* fields 'key' and 'value'.
*/
public class MapEntries extends ScalarFunction
- implements UnaryExpression, CustomSignature, PropagateNullable {
+ implements UnaryExpression, CustomSignature, PropagateNullable,
PreferPushDownProject {
/**
* constructor with 1 argument.
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
index 957dae2844a..a3123d29cb0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapKeys.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_keys'.
*/
public class MapKeys extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature,
PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new
FollowToAnyDataType(0))).args(MapType.of(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
index e7ca7f48dd3..1622b277156 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/MapValues.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import org.apache.doris.nereids.trees.expressions.functions.PropagateNullable;
import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression;
@@ -37,7 +38,7 @@ import java.util.List;
* ScalarFunction 'map_values'.
*/
public class MapValues extends ScalarFunction
- implements UnaryExpression, ExplicitlyCastableSignature,
PropagateNullable {
+ implements UnaryExpression, ExplicitlyCastableSignature,
PropagateNullable, PreferPushDownProject {
public static final List<FunctionSignature> SIGNATURES = ImmutableList.of(
FunctionSignature.ret(ArrayType.of(new
FollowToAnyDataType(0))).args(MapType.of(
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
index 85bb424de19..63169513423 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/StructElement.java
@@ -20,6 +20,7 @@ package
org.apache.doris.nereids.trees.expressions.functions.scalar;
import org.apache.doris.catalog.FunctionSignature;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.PreferPushDownProject;
import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable;
import
org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature;
import
org.apache.doris.nereids.trees.expressions.functions.PropagateNullLiteral;
@@ -40,7 +41,7 @@ import java.util.List;
* ScalarFunction 'struct_element'.
*/
public class StructElement extends ScalarFunction
- implements ExplicitlyCastableSignature, AlwaysNullable,
PropagateNullLiteral {
+ implements ExplicitlyCastableSignature, AlwaysNullable,
PropagateNullLiteral, PreferPushDownProject {
/**
* constructor with 0 or more arguments.
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
index 468167b68d4..096839b2e57 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PruneNestedColumnTest.java
@@ -28,10 +28,14 @@ import
org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
+import
org.apache.doris.nereids.trees.expressions.functions.scalar.StructElement;
+import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import org.apache.doris.nereids.trees.plans.physical.PhysicalCTEConsumer;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalUnion;
import org.apache.doris.nereids.types.DataType;
+import org.apache.doris.nereids.util.MemoPatternMatchSupported;
+import org.apache.doris.nereids.util.PlanChecker;
import org.apache.doris.planner.OlapScanNode;
import org.apache.doris.planner.PlanFragment;
import org.apache.doris.thrift.TAccessPathType;
@@ -52,7 +56,7 @@ import java.util.Map.Entry;
import java.util.TreeSet;
import java.util.function.Consumer;
-public class PruneNestedColumnTest extends TestWithFeService {
+public class PruneNestedColumnTest extends TestWithFeService implements
MemoPatternMatchSupported {
@BeforeAll
public void createTable() throws Exception {
createDatabase("test");
@@ -297,6 +301,59 @@ public class PruneNestedColumnTest extends
TestWithFeService {
);
}
+ @Test
+ public void testPushDownThroughJoin() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select * from
tbl)a join (select 100 id, 'f1' name)b on a.id=b.id")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalProject(
+ logicalJoin(
+ logicalProject(
+ logicalFilter(
+ logicalOlapScan()
+ )
+ ).when(p -> {
+ Assertions.assertEquals(2,
p.getProjects().size());
+
Assertions.assertTrue(p.getProjects().stream()
+ .anyMatch(o -> o instanceof Alias
&& o.child(0) instanceof StructElement));
+ return true;
+ }),
+ logicalOneRowRelation()
+ )
+ ).when(p -> {
+ Assertions.assertTrue(p.getProjects().size() == 1
&& p.getProjects().get(0) instanceof SlotReference);
+ return true;
+ })
+ )
+ );
+ }
+
+ @Test
+ public void testPushDownThroughUnion() {
+ PlanChecker.from(connectContext)
+ .analyze("select struct_element(s, 'city') from (select id, s
from tbl union all select 1, null) tmp")
+ .rewrite()
+ .matches(
+ logicalResultSink(
+ logicalUnion(
+ logicalProject(
+ logicalOlapScan()
+ ).when(p -> {
+ Assertions.assertEquals(1,
p.getProjects().size());
+
Assertions.assertInstanceOf(StructElement.class,
p.getProjects().get(0).child(0));
+ return true;
+ })
+ ).when(u -> {
+ Assertions.assertEquals(1,
u.getConstantExprsList().size());
+ Assertions.assertInstanceOf(NullLiteral.class,
u.getConstantExprsList().get(0).get(0).child(0));
+ return true;
+ })
+ )
+ );
+ }
+
private void assertColumn(String sql, String expectType,
List<TColumnNameAccessPath> expectAllAccessPaths,
List<TColumnNameAccessPath> expectPredicateAccessPaths) throws
Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
