This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.0 by this push:
new 9577291752c branch-4.0:[fix](partition_prune) Move the pruning of
predicates that are alwaystrue after partition pruning into the
PlanPostProcessor #63111 (#63467)
9577291752c is described below
commit 9577291752c2b18873d4c688f410ae9eac689248
Author: feiniaofeiafei <[email protected]>
AuthorDate: Fri May 22 14:20:50 2026 +0800
branch-4.0:[fix](partition_prune) Move the pruning of predicates that are
alwaystrue after partition pruning into the PlanPostProcessor #63111 (#63467)
picked from #63111
Co-authored-by: Copilot <[email protected]>
---
.../nereids/processor/post/PlanPostProcessors.java | 1 +
.../processor/post/PrunePartitionPredicate.java | 146 ++++++++++++
.../exploration/mv/SyncMaterializationContext.java | 11 +-
.../rules/expression/rules/PartitionPruner.java | 23 --
.../LogicalOlapScanToPhysicalOlapScan.java | 3 +-
.../rules/rewrite/PruneOlapScanPartition.java | 52 ++++-
.../trees/plans/PartitionPrunablePredicate.java | 95 ++++++++
.../trees/plans/logical/LogicalOlapScan.java | 90 +++++--
.../trees/plans/physical/PhysicalOlapScan.java | 97 ++++++--
.../nereids/rules/rewrite/PartitionPrunerTest.java | 258 ---------------------
.../partition_prune/prune_predicates_mv_test.out | 14 ++
.../prune_predicates_mv_test.groovy | 182 +++++++++++++++
12 files changed, 649 insertions(+), 323 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
index 621978495ca..dd33673144e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java
@@ -64,6 +64,7 @@ public class PlanPostProcessors {
// add processor if we need
Builder<PlanPostProcessor> builder = ImmutableList.builder();
builder.add(new PushDownFilterThroughProject());
+ builder.add(new PrunePartitionPredicate());
builder.add(new RemoveUselessProjectPostProcessor());
builder.add(new RecomputeLogicalPropertiesProcessor());
/*
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java
new file mode 100644
index 00000000000..9cb6ce33e7b
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PrunePartitionPredicate.java
@@ -0,0 +1,146 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.processor.post;
+
+import org.apache.doris.analysis.Expr;
+import org.apache.doris.analysis.SlotRef;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.physical.AbstractPhysicalPlan;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter;
+import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+
+/**
+ * Removes partition-prunable conjuncts that were registered by {@link
+ * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in
+ * the logical plan during cascades. Doing the removal here, after
+ * materialized-view rewrite has finished, ensures MV matching observes the
+ * original predicates; otherwise the MV view-predicate may incorrectly cover
+ * the dropped partition predicate and produce extra rows.
+ */
+public class PrunePartitionPredicate extends PlanPostProcessor {
+
+ @Override
+ public Plan visitPhysicalFilter(PhysicalFilter<? extends Plan> filter,
CascadesContext context) {
+ filter = (PhysicalFilter<? extends Plan>) super.visit(filter, context);
+ Plan child = filter.child();
+ if (!(child instanceof PhysicalOlapScan)) {
+ return filter;
+ }
+ PhysicalOlapScan scan = (PhysicalOlapScan) child;
+ Optional<PartitionPrunablePredicate> entryOpt =
scan.getPartitionPrunablePredicates();
+ if (!entryOpt.isPresent()) {
+ return filter;
+ }
+ boolean skipPrunePredicate =
context.getConnectContext().getSessionVariable().skipPrunePredicate
+ || context.getStatementContext().isSkipPrunePredicate();
+ if (skipPrunePredicate) {
+ return filter;
+ }
+ Set<Long> scanPartitions = new
HashSet<>(scan.getSelectedPartitionIds());
+ Map<String, Slot> nameToOutputSlot = buildNameToSlotMap(scan);
+
+ Set<Expression> remaining = new LinkedHashSet<>(filter.getConjuncts());
+ boolean changed = false;
+ PartitionPrunablePredicate entry = entryOpt.get();
+ if (entry.getSelectedPartitionIds().containsAll(scanPartitions)) {
+ Map<Expression, Expression> slotReplaceMap =
+ buildSlotReplaceMap(entry.getSnapshotPartitionSlots(),
nameToOutputSlot);
+ if (slotReplaceMap != null) {
+ for (Expression conjunct : entry.getPrunableConjuncts()) {
+ Expression rewritten = slotReplaceMap.isEmpty()
+ ? conjunct : ExpressionUtils.replace(conjunct,
slotReplaceMap);
+ if (remaining.remove(rewritten)) {
+ changed = true;
+ }
+ }
+ }
+ }
+ if (!changed) {
+ return filter;
+ }
+ if (remaining.isEmpty()) {
+ return scan;
+ }
+ return filter.withConjunctsAndChild(remaining, scan)
+ .copyStatsAndGroupIdFrom((AbstractPhysicalPlan) filter);
+ }
+
+ private static Map<String, Slot> buildNameToSlotMap(PhysicalOlapScan scan)
{
+ OlapTable table = scan.getTable();
+ List<Slot> slots = scan.getOutput();
+ Map<String, Slot> map = new HashMap<>(slots.size());
+ if (scan.getSelectedIndexId() == table.getBaseIndexId()) {
+ for (Slot slot : slots) {
+ map.put(slot.getName().toLowerCase(), slot);
+ }
+ } else {
+ for (Slot slot : slots) {
+ if (!(slot instanceof SlotReference)) {
+ continue;
+ }
+ SlotReference slotReference = (SlotReference) slot;
+ Optional<Column> columnOptional =
slotReference.getOriginalColumn();
+ if (!columnOptional.isPresent()) {
+ continue;
+ }
+ Expr expr = columnOptional.get().getDefineExpr();
+ if (!(expr instanceof SlotRef)) {
+ continue;
+ }
+ map.put(((SlotRef) expr).getColumnName().toLowerCase(), slot);
+ }
+ }
+ return map;
+ }
+
+ /**
+ * Map each recorded snapshot slot to the scan's current output slot of the
+ * same column name. Returns null when any snapshot slot cannot be located,
+ * so the caller can skip the entry.
+ */
+ private static Map<Expression, Expression> buildSlotReplaceMap(
+ List<Slot> snapshotSlots, Map<String, Slot> nameToOutputSlot) {
+ Map<Expression, Expression> replaceMap = new
HashMap<>(snapshotSlots.size());
+ for (Slot snapshot : snapshotSlots) {
+ Slot current =
nameToOutputSlot.get(snapshot.getName().toLowerCase());
+ if (current == null) {
+ return null;
+ }
+ if (!snapshot.equals(current)) {
+ replaceMap.put(snapshot, current);
+ }
+ }
+ return replaceMap;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
index 95c8d372665..0aa5108efdf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SyncMaterializationContext.java
@@ -122,13 +122,18 @@ public class SyncMaterializationContext extends
MaterializationContext {
return scanPlan.accept(new DefaultPlanRewriter<Void>() {
@Override
public Plan visitLogicalOlapScan(LogicalOlapScan olapScan,
Void context) {
- if
(!queryStructInfoRelations.get(0).getTable().getFullQualifiers().equals(
+ LogicalOlapScan queryScan = (LogicalOlapScan)
queryStructInfoRelations.get(0);
+ if (!queryScan.getTable().getFullQualifiers().equals(
olapScan.getTable().getFullQualifiers())) {
// Only the same table, we can do partition prue
return olapScan;
}
- return olapScan.withSelectedPartitionIds(
- ((LogicalOlapScan)
queryStructInfoRelations.get(0)).getSelectedPartitionIds());
+ // Carry partition-prunable predicates from the original
query scan onto
+ // the rewritten MV scan so the post-processor can still
drop the
+ // predicates that have already been enforced by partition
pruning.
+ return olapScan
+
.withSelectedPartitionIds(queryScan.getSelectedPartitionIds())
+
.withPartitionPrunablePredicates(queryScan.getPartitionPrunablePredicates());
}
}, null);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
index badc925ad38..eea4aeff433 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java
@@ -36,11 +36,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.DateLiteral;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
import org.apache.doris.nereids.types.DateTimeType;
-import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import com.google.common.collect.ImmutableList;
@@ -51,7 +47,6 @@ import com.google.common.collect.Range;
import com.google.common.collect.RangeSet;
import com.google.common.collect.Sets;
-import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -355,22 +350,4 @@ public class PartitionPruner extends
DefaultExpressionRewriter<Void> {
return Pair.of(true, false);
}
}
-
- /** remove predicates that are always true*/
- public static Plan prunePredicate(boolean skipPrunePredicate,
Optional<Expression> prunedPredicates,
- LogicalFilter<? extends Plan> filter, LogicalRelation scan) {
- if (!skipPrunePredicate && prunedPredicates.isPresent()) {
- Set<Expression> conjuncts = new
LinkedHashSet<>(filter.getConjuncts());
- Expression deletedPredicate = prunedPredicates.get();
- Set<Expression> deletedPredicateSet =
ExpressionUtils.extractConjunctionToSet(deletedPredicate);
- conjuncts.removeAll(deletedPredicateSet);
- if (conjuncts.isEmpty()) {
- return scan;
- } else {
- return filter.withConjunctsAndChild(conjuncts, scan);
- }
- } else {
- return filter.withChildren(ImmutableList.of(scan));
- }
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
index 4a7132c3a85..bb5f5a8b97c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java
@@ -68,7 +68,8 @@ public class LogicalOlapScanToPhysicalOlapScan extends
OneImplementationRuleFact
olapScan.getScoreLimit(),
olapScan.getScoreRangeInfo(),
olapScan.getAnnOrderKeys(),
- olapScan.getAnnLimit())
+ olapScan.getAnnLimit(),
+ olapScan.getPartitionPrunablePredicates())
).toRule(RuleType.LOGICAL_OLAP_SCAN_TO_PHYSICAL_OLAP_SCAN_RULE);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
index 937f7ed5d50..9d55e1c0c3b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneOlapScanPartition.java
@@ -35,10 +35,12 @@ import
org.apache.doris.nereids.rules.expression.rules.PartitionPruner.Partition
import org.apache.doris.nereids.rules.expression.rules.SortedPartitionRanges;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRelation;
+import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.nereids.util.Utils;
import org.apache.doris.qe.ConnectContext;
@@ -46,6 +48,7 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableSet;
import java.util.ArrayList;
+import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
@@ -89,12 +92,29 @@ public class PruneOlapScanPartition implements
RewriteRuleFactory {
}
if (rewrittenLogicalRelation instanceof
LogicalEmptyRelation) {
return rewrittenLogicalRelation;
- } else {
- return PartitionPruner.prunePredicate(
-
ctx.connectContext.getSessionVariable().skipPrunePredicate
- ||
ctx.statementContext.isSkipPrunePredicate(),
- prunedRes.second, filter,
rewrittenLogicalRelation);
}
+ boolean skipPrunePredicate =
ctx.connectContext.getSessionVariable().skipPrunePredicate
+ || ctx.statementContext.isSkipPrunePredicate();
+ if (!skipPrunePredicate &&
prunedRes.second.isPresent()) {
+ // Defer the predicate removal to
PlanPostProcessor so that materialized-view
+ // rewrite still sees the original predicates.
Otherwise, partition predicates
+ // that are equivalent to the surviving partition
list would be silently
+ // dropped, leading to wrong results when an MV
definition predicate matches
+ // the remaining conjuncts.
+ LogicalOlapScan prunedScan = (LogicalOlapScan)
rewrittenLogicalRelation;
+ Set<Expression> prunableConjuncts =
ExpressionUtils.extractConjunctionToSet(
+ prunedRes.second.get());
+ List<Slot> partitionSlots =
getPartitionSlots(prunedScan, prunedScan.getTable());
+ if (partitionSlots != null) {
+ PartitionPrunablePredicate entry = new
PartitionPrunablePredicate(
+ new
HashSet<>(prunedScan.getSelectedPartitionIds()),
+ partitionSlots,
+ prunableConjuncts);
+ rewrittenLogicalRelation =
prunedScan.withPartitionPrunablePredicates(
+ Optional.of(entry));
+ }
+ }
+ return
filter.withChildren(ImmutableList.of(rewrittenLogicalRelation));
}).toRule(RuleType.OLAP_SCAN_PARTITION_PRUNE)
);
}
@@ -174,6 +194,28 @@ public class PruneOlapScanPartition implements
RewriteRuleFactory {
}
}
+ private List<Slot> getPartitionSlots(LogicalOlapScan scan, OlapTable
table) {
+ List<Slot> output = scan.getOutput();
+ PartitionInfo partitionInfo = table.getPartitionInfo();
+ List<Column> partitionColumns = partitionInfo.getPartitionColumns();
+ List<Slot> partitionSlots = new ArrayList<>(partitionColumns.size());
+ for (Column column : partitionColumns) {
+ Slot partitionSlot = null;
+ // loop search is faster than build a map
+ for (Slot slot : output) {
+ if (slot.getName().equalsIgnoreCase(column.getName())) {
+ partitionSlot = slot;
+ break;
+ }
+ }
+ if (partitionSlot == null) {
+ return null;
+ }
+ partitionSlots.add(partitionSlot);
+ }
+ return partitionSlots;
+ }
+
private List<Long> prunePartitionByTabletIds(LogicalOlapScan scan,
OlapTable table,
List<Long>
prunedPartitionsByFilters) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java
new file mode 100644
index 00000000000..54ba19493b8
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PartitionPrunablePredicate.java
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.trees.plans;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.Slot;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
+
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+
+/**
+ * Records that, on the scan whose partition list equals {@link
+ * #selectedPartitionIds}, the {@link #prunableConjuncts} are guaranteed to
+ * evaluate to TRUE for every surviving row.
+ *
+ * <p>The predicate is registered by {@link
+ * org.apache.doris.nereids.rules.rewrite.PruneOlapScanPartition} but kept in
+ * the logical filter during cascades. The actual removal happens later in
+ * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate} so
+ * that materialized-view rewrite still sees the original predicates. Keeping
+ * the predicate in the plan avoids the wrong-result problem in which the MV
+ * view-predicate happens to cover the remaining conjuncts after the partition
+ * predicate has been silently dropped.
+ *
+ * <p>The predicate lives on the scan itself (see {@code LogicalOlapScan} and
+ * {@code PhysicalOlapScan}) so we no longer need to match it back to its scan
+ * via a table identifier. Because rewrites between recording and removal may
+ * rebuild the scan with fresh slot ids, {@link #snapshotPartitionSlots}
+ * captures the slots that appear in the recorded conjuncts. The post-processor
+ * maps them onto the actual scan's output slots by column name before
+ * performing the conjunct removal.
+ */
+public class PartitionPrunablePredicate {
+ private final Set<Long> selectedPartitionIds;
+ private final List<Slot> snapshotPartitionSlots;
+ private final Set<Expression> prunableConjuncts;
+
+ public PartitionPrunablePredicate(Set<Long> selectedPartitionIds,
+ List<Slot> snapshotPartitionSlots,
+ Set<Expression> prunableConjuncts) {
+ this.selectedPartitionIds = ImmutableSet.copyOf(selectedPartitionIds);
+ this.snapshotPartitionSlots =
ImmutableList.copyOf(snapshotPartitionSlots);
+ this.prunableConjuncts = ImmutableSet.copyOf(prunableConjuncts);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ PartitionPrunablePredicate that = (PartitionPrunablePredicate) o;
+ return selectedPartitionIds.equals(that.selectedPartitionIds)
+ && snapshotPartitionSlots.equals(that.snapshotPartitionSlots)
+ && prunableConjuncts.equals(that.prunableConjuncts);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(selectedPartitionIds, snapshotPartitionSlots,
prunableConjuncts);
+ }
+
+ public Set<Long> getSelectedPartitionIds() {
+ return selectedPartitionIds;
+ }
+
+ public List<Slot> getSnapshotPartitionSlots() {
+ return snapshotPartitionSlots;
+ }
+
+ public Set<Expression> getPrunableConjuncts() {
+ return prunableConjuncts;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
index 2ab3405f3c0..2cf3555a7de 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java
@@ -34,6 +34,7 @@ import
org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@@ -159,6 +160,16 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
private final List<OrderKey> annOrderKeys;
private final Optional<Long> annLimit;
+ /**
+ * Conjuncts that are guaranteed to be TRUE on the current scan because
they
+ * were already enforced by partition pruning. The deferred removal happens
+ * in the {@link
org.apache.doris.nereids.processor.post.PrunePartitionPredicate}
+ * post-processor so that materialized-view rewrite still observes the
+ * original predicates. The set is preserved through {@code with*} rewrites
+ * and copied onto MV rewrite outputs.
+ */
+ private final Optional<PartitionPrunablePredicate>
partitionPrunablePredicates;
+
public LogicalOlapScan(RelationId id, OlapTable table) {
this(id, table, ImmutableList.of());
}
@@ -233,6 +244,29 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
List<OrderKey> annOrderKeys, Optional<Long> annLimit) {
+ this(id, table, qualifier, groupExpression, logicalProperties,
selectedPartitionIds, partitionPruned,
+ selectedTabletIds, selectedIndexId, indexSelected,
preAggStatus,
+ specifiedPartitions, hints, cacheSlotWithSlotName,
cachedOutput, tableSample, directMvScan,
+ colToSubPathsMap, specifiedTabletIds, operativeSlots,
virtualColumns,
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit,
+ Optional.empty());
+ }
+
+ /**
+ * Constructor for LogicalOlapScan.
+ */
+ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier,
+ Optional<GroupExpression> groupExpression,
Optional<LogicalProperties> logicalProperties,
+ List<Long> selectedPartitionIds, boolean partitionPruned,
+ List<Long> selectedTabletIds, long selectedIndexId, boolean
indexSelected,
+ PreAggStatus preAggStatus, List<Long> specifiedPartitions,
+ List<String> hints, Map<Pair<Long, String>, Slot>
cacheSlotWithSlotName,
+ Optional<List<Slot>> cachedOutput, Optional<TableSample>
tableSample, boolean directMvScan,
+ Map<String, Set<List<String>>> colToSubPathsMap, List<Long>
specifiedTabletIds,
+ Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
+ List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
+ List<OrderKey> annOrderKeys, Optional<Long> annLimit,
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier,
operativeSlots, virtualColumns, groupExpression,
logicalProperties);
Preconditions.checkArgument(selectedPartitionIds != null,
@@ -270,12 +304,37 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
this.scoreRangeInfo = scoreRangeInfo;
this.annOrderKeys = Utils.fastToImmutableList(annOrderKeys);
this.annLimit = annLimit;
+ this.partitionPrunablePredicates = partitionPrunablePredicates == null
+ ? Optional.empty()
+ : partitionPrunablePredicates;
}
public List<Long> getSelectedPartitionIds() {
return selectedPartitionIds;
}
+ public Optional<PartitionPrunablePredicate>
getPartitionPrunablePredicates() {
+ return partitionPrunablePredicates;
+ }
+
+ /**
+ * Returns a new {@code LogicalOlapScan} carrying the supplied
+ * {@link PartitionPrunablePredicate}. It is preserved across all other
+ * {@code with*} builders so partition-derived conjuncts can be removed
+ * safely after MV rewrite has had a chance to match the plan.
+ */
+ public LogicalOlapScan withPartitionPrunablePredicates(
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
+ return new LogicalOlapScan(relationId, (Table) table, qualifier,
+ groupExpression, Optional.of(getLogicalProperties()),
+ selectedPartitionIds, partitionPruned, selectedTabletIds,
+ selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
+ hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
+ colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit,
+ partitionPrunablePredicates);
+ }
+
@Override
public String getFingerprint() {
String partitions = "";
@@ -340,7 +399,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
&& Objects.equals(scoreLimit, that.scoreLimit)
&& Objects.equals(scoreRangeInfo, that.scoreRangeInfo)
&& Objects.equals(annOrderKeys, that.annOrderKeys)
- && Objects.equals(annLimit, that.annLimit);
+ && Objects.equals(annLimit, that.annLimit)
+ && Objects.equals(partitionPrunablePredicates,
that.partitionPrunablePredicates);
}
@Override
@@ -356,7 +416,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, partitionPrunablePredicates);
}
@Override
@@ -367,7 +427,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, partitionPrunablePredicates);
}
/**
@@ -380,7 +440,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, partitionPrunablePredicates);
}
/**
@@ -395,7 +455,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
indexId, true, PreAggStatus.unset(),
manuallySpecifiedPartitions, hints, cacheSlotWithSlotName,
cachedOutput, tableSample, directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds,
operativeSlots, virtualColumns, scoreOrderKeys, scoreLimit,
scoreRangeInfo,
- annOrderKeys, annLimit);
+ annOrderKeys, annLimit, partitionPrunablePredicates);
}
/**
@@ -408,7 +468,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns, scoreOrderKeys,
- scoreLimit, scoreRangeInfo, annOrderKeys, annLimit);
+ scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
/**
@@ -421,7 +481,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, partitionPrunablePredicates);
}
/**
@@ -434,7 +494,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, partitionPrunablePredicates);
}
/**
@@ -447,7 +507,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan,
colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots,
virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit, partitionPrunablePredicates);
}
@Override
@@ -459,7 +519,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, Maps.newHashMap(), Optional.empty(), tableSample,
directMvScan,
colToSubPathsMap, selectedTabletIds, operativeSlots,
virtualColumns, scoreOrderKeys,
- scoreLimit, scoreRangeInfo, annOrderKeys, annLimit);
+ scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
/**
@@ -479,7 +539,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit);
+ scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
/**
@@ -502,7 +562,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots,
mergedVirtualColumns, scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit);
+ scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
/**
@@ -531,7 +591,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots,
mergedVirtualColumns, scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit);
+ scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
@Override
@@ -870,7 +930,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation
implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, cachedOutput, tableSample,
directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit);
+ scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
@VisibleForTesting
@@ -951,7 +1011,7 @@ public class LogicalOlapScan extends
LogicalCatalogRelation implements OlapScan,
selectedIndexId, indexSelected, preAggStatus,
manuallySpecifiedPartitions,
hints, cacheSlotWithSlotName, Optional.of(outputSlots),
tableSample, directMvScan, colToSubPathsMap,
manuallySpecifiedTabletIds, operativeSlots, virtualColumns,
scoreOrderKeys, scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit);
+ scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
index 0b16789bd2b..5644ca25e5b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java
@@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.Slot;
+import org.apache.doris.nereids.trees.plans.PartitionPrunablePredicate;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.PlanType;
import org.apache.doris.nereids.trees.plans.PreAggStatus;
@@ -76,6 +77,15 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
private final List<OrderKey> annOrderKeys;
private final Optional<Long> annLimit;
+ /**
+ * Predicates known to be TRUE on this scan thanks to partition pruning.
+ * Carried alongside the scan so the
+ * {@link org.apache.doris.nereids.processor.post.PrunePartitionPredicate}
+ * post-processor can strip them from the surrounding filter after MV
+ * rewrite has finished its matching work.
+ */
+ private final Optional<PartitionPrunablePredicate>
partitionPrunablePredicates;
+
/**
* Constructor for PhysicalOlapScan.
*/
@@ -101,11 +111,50 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
List<Long> selectedTabletIds, List<Long> selectedPartitionIds,
DistributionSpec distributionSpec,
PreAggStatus preAggStatus, List<Slot> baseOutputs,
Optional<GroupExpression> groupExpression, LogicalProperties
logicalProperties,
+ Optional<TableSample> tableSample, List<Slot> operativeSlots,
List<NamedExpression> virtualColumns,
+ List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
+ List<OrderKey> annOrderKeys, Optional<Long> annLimit,
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
+ this(id, olapTable, qualifier,
+ selectedIndexId, selectedTabletIds, selectedPartitionIds,
distributionSpec,
+ preAggStatus, baseOutputs,
+ groupExpression, logicalProperties, null,
+ null, tableSample, operativeSlots, virtualColumns,
scoreOrderKeys, scoreLimit,
+ scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
+ }
+
+ /**
+ * Constructor for PhysicalOlapScan.
+ */
+ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List<String>
qualifier, long selectedIndexId,
+ List<Long> selectedTabletIds, List<Long> selectedPartitionIds,
+ DistributionSpec distributionSpec, PreAggStatus preAggStatus,
List<Slot> baseOutputs,
+ Optional<GroupExpression> groupExpression, LogicalProperties
logicalProperties,
PhysicalProperties physicalProperties, Statistics statistics,
Optional<TableSample> tableSample,
Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
List<OrderKey> annOrderKeys, Optional<Long> annLimit) {
+ this(id, olapTable, qualifier, selectedIndexId, selectedTabletIds,
selectedPartitionIds,
+ distributionSpec, preAggStatus, baseOutputs, groupExpression,
+ logicalProperties, physicalProperties, statistics,
tableSample, operativeSlots, virtualColumns,
+ scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit,
+ Optional.empty());
+ }
+
+ /**
+ * Ultimate constructor for PhysicalOlapScan.
+ */
+ public PhysicalOlapScan(RelationId id, OlapTable olapTable, List<String>
qualifier, long selectedIndexId,
+ List<Long> selectedTabletIds, List<Long> selectedPartitionIds,
+ DistributionSpec distributionSpec, PreAggStatus preAggStatus,
List<Slot> baseOutputs,
+ Optional<GroupExpression> groupExpression, LogicalProperties
logicalProperties,
+ PhysicalProperties physicalProperties, Statistics statistics,
+ Optional<TableSample> tableSample,
+ Collection<Slot> operativeSlots, List<NamedExpression>
virtualColumns,
+ List<OrderKey> scoreOrderKeys, Optional<Long> scoreLimit,
Optional<ScoreRangeInfo> scoreRangeInfo,
+ List<OrderKey> annOrderKeys, Optional<Long> annLimit,
+ Optional<PartitionPrunablePredicate> partitionPrunablePredicates) {
super(id, PlanType.PHYSICAL_OLAP_SCAN, olapTable, qualifier,
groupExpression, logicalProperties, physicalProperties,
statistics, operativeSlots);
this.selectedIndexId = selectedIndexId;
@@ -122,6 +171,9 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
this.scoreRangeInfo = scoreRangeInfo;
this.annOrderKeys = ImmutableList.copyOf(annOrderKeys);
this.annLimit = annLimit;
+ this.partitionPrunablePredicates = partitionPrunablePredicates == null
+ ? Optional.empty()
+ : partitionPrunablePredicates;
}
@Override
@@ -138,6 +190,10 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
return selectedPartitionIds;
}
+ public Optional<PartitionPrunablePredicate>
getPartitionPrunablePredicates() {
+ return partitionPrunablePredicates;
+ }
+
@Override
public OlapTable getTable() {
return (OlapTable) table;
@@ -266,7 +322,8 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
&& Objects.equals(scoreLimit, olapScan.scoreLimit)
&& Objects.equals(scoreRangeInfo, olapScan.scoreRangeInfo)
&& Objects.equals(annOrderKeys, olapScan.annOrderKeys)
- && Objects.equals(annLimit, olapScan.annLimit);
+ && Objects.equals(annLimit, olapScan.annLimit)
+ && Objects.equals(partitionPrunablePredicates,
olapScan.partitionPrunablePredicates);
}
@Override
@@ -281,28 +338,31 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
@Override
public PhysicalOlapScan withGroupExpression(Optional<GroupExpression>
groupExpression) {
- return new PhysicalOlapScan(relationId, getTable(), qualifier,
selectedIndexId, selectedTabletIds,
- selectedPartitionIds, distributionSpec, preAggStatus,
baseOutputs,
- groupExpression, getLogicalProperties(), null, null,
tableSample, operativeSlots, virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ return new PhysicalOlapScan(relationId, getTable(), qualifier,
+ selectedIndexId, selectedTabletIds, selectedPartitionIds,
+ distributionSpec, preAggStatus, baseOutputs, groupExpression,
getLogicalProperties(), null, null,
+ tableSample, operativeSlots, virtualColumns, scoreOrderKeys,
scoreLimit, scoreRangeInfo,
+ annOrderKeys, annLimit, partitionPrunablePredicates);
}
@Override
public Plan withGroupExprLogicalPropChildren(Optional<GroupExpression>
groupExpression,
Optional<LogicalProperties> logicalProperties, List<Plan>
children) {
- return new PhysicalOlapScan(relationId, getTable(), qualifier,
selectedIndexId, selectedTabletIds,
- selectedPartitionIds, distributionSpec, preAggStatus,
baseOutputs, groupExpression,
- logicalProperties.get(), null, null, tableSample,
operativeSlots, virtualColumns,
- scoreOrderKeys, scoreLimit, scoreRangeInfo, annOrderKeys,
annLimit);
+ return new PhysicalOlapScan(relationId, getTable(), qualifier,
+ selectedIndexId, selectedTabletIds, selectedPartitionIds,
+ distributionSpec, preAggStatus, baseOutputs, groupExpression,
logicalProperties.get(), null, null,
+ tableSample, operativeSlots, virtualColumns, scoreOrderKeys,
scoreLimit, scoreRangeInfo,
+ annOrderKeys, annLimit, partitionPrunablePredicates);
}
@Override
public PhysicalOlapScan withPhysicalPropertiesAndStats(
PhysicalProperties physicalProperties, Statistics statistics) {
- return new PhysicalOlapScan(relationId, getTable(), qualifier,
selectedIndexId, selectedTabletIds,
- selectedPartitionIds, distributionSpec, preAggStatus,
baseOutputs, groupExpression,
- getLogicalProperties(), physicalProperties, statistics,
tableSample, operativeSlots,
- virtualColumns, scoreOrderKeys, scoreLimit, scoreRangeInfo,
annOrderKeys, annLimit);
+ return new PhysicalOlapScan(relationId, getTable(), qualifier,
+ selectedIndexId, selectedTabletIds, selectedPartitionIds,
+ distributionSpec, preAggStatus, baseOutputs, groupExpression,
getLogicalProperties(),
+ physicalProperties, statistics, tableSample, operativeSlots,
virtualColumns, scoreOrderKeys,
+ scoreLimit, scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
@Override
@@ -325,11 +385,12 @@ public class PhysicalOlapScan extends
PhysicalCatalogRelation implements OlapSca
@Override
public CatalogRelation withOperativeSlots(Collection<Slot> operativeSlots)
{
- return new PhysicalOlapScan(relationId, (OlapTable) table, qualifier,
selectedIndexId, selectedTabletIds,
- selectedPartitionIds, distributionSpec, preAggStatus,
baseOutputs,
- groupExpression, getLogicalProperties(),
getPhysicalProperties(), statistics,
- tableSample, operativeSlots, virtualColumns, scoreOrderKeys,
scoreLimit,
- scoreRangeInfo, annOrderKeys, annLimit);
+ return new PhysicalOlapScan(relationId, (OlapTable) table, qualifier,
+ selectedIndexId, selectedTabletIds, selectedPartitionIds,
+ distributionSpec, preAggStatus, baseOutputs, groupExpression,
getLogicalProperties(),
+ getPhysicalProperties(), statistics, tableSample,
operativeSlots, virtualColumns, scoreOrderKeys,
+ scoreLimit,
+ scoreRangeInfo, annOrderKeys, annLimit,
partitionPrunablePredicates);
}
@Override
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
index 0378cc86dba..116d2a89272 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PartitionPrunerTest.java
@@ -20,7 +20,6 @@ package org.apache.doris.nereids.rules.rewrite;
import org.apache.doris.analysis.PartitionValue;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.ListPartitionItem;
-import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionKey;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.common.AnalysisException;
@@ -34,16 +33,11 @@ import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.GreaterThan;
import org.apache.doris.nereids.trees.expressions.InPredicate;
-import org.apache.doris.nereids.trees.expressions.LessThan;
import org.apache.doris.nereids.trees.expressions.Not;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.RelationId;
-import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.types.VarcharType;
import org.apache.doris.utframe.TestWithFeService;
@@ -54,10 +48,7 @@ import org.junit.jupiter.api.Test;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
-import java.util.LinkedHashSet;
import java.util.List;
-import java.util.Optional;
-import java.util.Set;
public class PartitionPrunerTest extends TestWithFeService {
private Method canBePrunedOutMethod;
@@ -317,254 +308,5 @@ public class PartitionPrunerTest extends
TestWithFeService {
Assertions.assertFalse(result.first);
Assertions.assertFalse(result.second);
}
-
- // test prune predicate
- // Test basis: some predicates are pruned
- @Test
- public void testPrunePartialPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- predicates.add(gt);
- predicates.add(lt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
- }
-
- // all predicates are pruned
- @Test
- public void testPruneAllPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- predicates.add(gt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
- }
-
- // no predicates are pruned
- @Test
- public void testPruneNoPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- predicates.add(gt);
- predicates.add(lt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- EqualTo nonExistentPredicate = new EqualTo(slotC, Literal.of(30));
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(nonExistentPredicate), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(2, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(lt));
- }
-
- @Test
- public void testPruneCompoundPredicate() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- EqualTo eq = new EqualTo(slotC, Literal.of(30));
- predicates.add(gt);
- predicates.add(lt);
- predicates.add(eq);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // (a > 10 AND b < 20)
- And compoundPredicate = new And(gt, lt);
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(compoundPredicate), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt));
- }
-
- @Test
- public void testSkipPrunePredicate() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- predicates.add(gt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(true,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
- }
-
- @Test
- public void testEmptyPrunedPredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- predicates.add(gt);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // prunedPredicates is empty
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.empty(), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(gt));
- }
-
- @Test
- public void testPruneDuplicatePredicates() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt1 = new GreaterThan(slotA, Literal.of(10));
- GreaterThan gt2 = new GreaterThan(slotA, Literal.of(10)); //
duplicated predicate
- predicates.add(gt1);
- predicates.add(gt2);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt1), filter, scan);
-
- Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
- }
-
- @Test
- public void testPruneWithNullLiteral() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- EqualTo nullEq = new EqualTo(slotB, new NullLiteral());
- predicates.add(gt);
- predicates.add(nullEq);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(nullEq));
- }
-
- @Test
- public void testPruneMultiplePredicatesPartially() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- EqualTo eq1 = new EqualTo(slotC, Literal.of(30));
- EqualTo eq2 = new EqualTo(slotC, Literal.of(40));
- predicates.add(gt);
- predicates.add(lt);
- predicates.add(eq1);
- predicates.add(eq2);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // (a > 10 AND b < 20)
- And compoundPredicate = new And(gt, lt);
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(compoundPredicate), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(2, prunedFilter.getConjuncts().size());
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq1));
- Assertions.assertTrue(prunedFilter.getConjuncts().contains(eq2));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(gt));
- Assertions.assertFalse(prunedFilter.getConjuncts().contains(lt));
- }
-
- @Test
- public void testPruneNestedCompoundPredicate() {
- Set<Expression> predicates = new LinkedHashSet<>();
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- LessThan lt = new LessThan(slotB, Literal.of(20));
- EqualTo eq = new EqualTo(slotC, Literal.of(30));
- predicates.add(gt);
- predicates.add(lt);
- predicates.add(eq);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- // (a > 10 AND (b < 20 AND c = 30))
- And innerAnd = new And(lt, eq);
- And outerAnd = new And(gt, innerAnd);
-
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(outerAnd), filter, scan);
-
- Assertions.assertInstanceOf(LogicalOlapScan.class, prunedPlan);
- }
-
- @Test
- public void testPruneWhenFilterContainsOr() {
- Set<Expression> predicates = new LinkedHashSet<>();
- Or orPredicate = new Or(
- new GreaterThan(slotA, Literal.of(10)),
- new LessThan(slotB, Literal.of(20))
- );
- predicates.add(orPredicate);
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
-
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
-
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
-
Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate));
- }
-
- @Test
- public void testPruneWhenFilterContainsAndOrMix() {
- Set<Expression> predicates = new LinkedHashSet<>();
- // filter :a > 10 AND (b < 20 OR c = 30)
- Or orPredicate = new Or(
- new LessThan(slotB, Literal.of(20)),
- new EqualTo(slotC, Literal.of(30))
- );
- GreaterThan gt = new GreaterThan(slotA, Literal.of(10));
-
- predicates.add(gt);
- predicates.add(orPredicate);
-
- LogicalOlapScan scan = new LogicalOlapScan(new RelationId(1), new
OlapTable());
- LogicalFilter<Plan> filter = new LogicalFilter<>(predicates, scan);
- // a > 10
- Plan prunedPlan = PartitionPruner.prunePredicate(false,
Optional.of(gt), filter, scan);
- Assertions.assertInstanceOf(LogicalFilter.class, prunedPlan);
- LogicalFilter<?> prunedFilter = (LogicalFilter<?>) prunedPlan;
-
- Assertions.assertEquals(1, prunedFilter.getConjuncts().size());
-
Assertions.assertTrue(prunedFilter.getConjuncts().contains(orPredicate));
- }
}
-
diff --git
a/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out
b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out
new file mode 100644
index 00000000000..cc704a5c7fe
--- /dev/null
+++
b/regression-test/data/nereids_rules_p0/partition_prune/prune_predicates_mv_test.out
@@ -0,0 +1,14 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !mv_1 --
+1
+2
+
+-- !mv_2 --
+1
+2
+
+-- !query_3 --
+a 1 5
+a 2 7
+a 3 0
+
diff --git
a/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy
b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy
new file mode 100644
index 00000000000..cf33f0a5f1d
--- /dev/null
+++
b/regression-test/suites/nereids_rules_p0/partition_prune/prune_predicates_mv_test.groovy
@@ -0,0 +1,182 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("prune_predicates_mv_test") {
+ String currentDb = context.config.getDbNameByFile(context.file)
+ sql """
+ drop table if exists base_t;
+ CREATE TABLE base_t (
+ top_asset varchar(64) NOT NULL,
+ tag_key int NOT NULL,
+ tag_value int NOT NULL,
+ frame_count int NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(top_asset, tag_key, tag_value)
+ AUTO PARTITION BY LIST (tag_key) ()
+ DISTRIBUTED BY HASH(top_asset) BUCKETS 4
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+
+ INSERT INTO base_t VALUES
+ ('a', 1, 100, 5), ('a', 1, 101, 0),
+ ('a', 2, 200, 7), ('a', 2, 201, 0),
+ ('a', 3, 300, 0),
+ ('a', 4, 400, 9),
+ ('a', 5, 500, 1),
+ ('a', 6, 600, 2);
+ """
+
+ // case 1:
+ def mv_1 = """
+ SELECT top_asset, tag_key, SUM(frame_count) AS frame_count
+ FROM base_t
+ WHERE frame_count != 0
+ GROUP BY top_asset, tag_key;
+ """
+
+ def query_1 = """
+ SELECT /*+ USE_MV(mv_1) */ tag_key FROM base_t
+ WHERE tag_key IN (1, 2, 3) AND frame_count != 0
+ GROUP BY tag_key
+ ORDER BY tag_key;
+ """
+
+ //Execute (force rewrite):
+ //1. verify result is correct
+ //2. verify shape plan contains filter
+ //3. verify rewrite succeeded: async_mv_rewrite_success
+
+ // The base table is partitioned and the predicate was removed after
partition prune;
+ // the MV is not partitioned, so verify the MV did not drop the predicate.
+ async_mv_rewrite_success(currentDb, mv_1, query_1, "mv_1")
+ order_qt_mv_1 query_1
+ explain {
+ sql "shape plan ${query_1}"
+ contains "filter"
+ }
+
+ // case2: the MV is also partitioned; verify that partition pruning on the
MV
+ // is performed and the always-true predicate is removed after the prune.
+
+ def async_partition_mv_rewrite_success = { db, mv_sql, query_sql, mv_name,
partition, expected_pre_rewrite_strategys = [] ->
+ if (!mvShouldContinueCheck(expected_pre_rewrite_strategys)) {
+ return;
+ }
+ sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}"""
+ sql"""
+ CREATE MATERIALIZED VIEW ${mv_name}
+ BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL
+ ${partition}
+ DISTRIBUTED BY RANDOM BUCKETS 2
+ PROPERTIES ('replication_num' = '1')
+ AS ${mv_sql}
+ """
+ def job_name = getJobName(db, mv_name);
+ waitingMTMVTaskFinished(job_name)
+ // force meta sync to avoid stale meta data on follower fe
+ sql """sync;"""
+ mv_rewrite_success(query_sql, mv_name, true,
expected_pre_rewrite_strategys)
+ }
+
+ sql """
+ drop table if exists base_t2;
+ CREATE TABLE base_t2 (
+ top_asset varchar(64) NOT NULL,
+ tag_key int NOT NULL,
+ tag_value int NOT NULL,
+ frame_count int NOT NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(top_asset, tag_key, tag_value)
+ AUTO PARTITION BY LIST (tag_key) ()
+ DISTRIBUTED BY HASH(top_asset) BUCKETS 4
+ PROPERTIES (
+ "enable_unique_key_merge_on_write" = "true",
+ "replication_num" = "1"
+ );
+
+ INSERT INTO base_t2 VALUES
+ ('a', 1, 100, 5), ('a', 1, 101, 0),
+ ('a', 2, 200, 7), ('a', 2, 201, 0),
+ ('a', 3, 300, 0),
+ ('a', 4, 400, 9),
+ ('a', 5, 500, 1),
+ ('a', 6, 600, 2);
+ """
+ def mv_2 = """
+ SELECT top_asset, tag_key, SUM(frame_count) AS frame_count
+ FROM base_t2
+ WHERE frame_count != 0
+ GROUP BY top_asset, tag_key;
+ """
+ def query_2 = """
+ SELECT /*+use_mv(mv_2)*/ tag_key FROM base_t2
+ WHERE tag_key IN (1, 2, 3) AND frame_count != 0
+ GROUP BY tag_key
+ ORDER BY tag_key;
+ """
+
+ async_partition_mv_rewrite_success(currentDb, mv_2, query_2, "mv_2",
"PARTITION BY (tag_key)")
+ order_qt_mv_2 query_2
+ explain {
+ sql "physical plan ${query_2}"
+ contains "partitions(2/6)"
+ notContains "PhysicalFilter"
+ }
+
+ sql """
+ drop table if exists base_t3;
+ CREATE TABLE base_t3 (
+ top_asset varchar(64) NOT NULL,
+ tag_key int NOT NULL,
+ tag_value int NOT NULL,
+ frame_count int NOT NULL
+ ) ENGINE=OLAP
+ duplicate KEY(top_asset, tag_key, tag_value)
+ AUTO PARTITION BY LIST (tag_key) ()
+ DISTRIBUTED BY HASH(top_asset) BUCKETS 4
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+
+ INSERT INTO base_t3 VALUES
+ ('a', 1, 100, 5), ('a', 1, 101, 0),
+ ('a', 2, 200, 7), ('a', 2, 201, 0),
+ ('a', 3, 300, 0),
+ ('a', 4, 400, 9),
+ ('a', 5, 500, 1),
+ ('a', 6, 600, 2);
+ """
+ create_sync_mv(currentDb, "base_t3", "mv_3", """
+ SELECT top_asset as mv_ta, tag_key as mv_tk, SUM(frame_count) AS mv_sum
+ FROM base_t3
+ GROUP BY top_asset, tag_key;
+ """)
+
+ def query_3 = """
+ SELECT top_asset as mv_ta, tag_key as mv_tk, SUM(frame_count) AS mv_sum
+ FROM base_t3
+ where tag_key in (1,2,3)
+ GROUP BY top_asset, tag_key;
+ """
+ mv_rewrite_success(query_3, "mv_3")
+ order_qt_query_3 query_3
+ explain {
+ sql "physical plan ${query_3}"
+ notContains "PhysicalFilter"
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
