This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new c3c6e91548c branch-4.2: [fix](mtmv): Re-prune Iceberg scans after
partition compensation #69678 (#70094)
c3c6e91548c is described below
commit c3c6e91548c548401e895fb510d9ecc21eef0e0a
Author: morrySnow <[email protected]>
AuthorDate: Thu Nov 5 09:27:57 2026 +0800
branch-4.2: [fix](mtmv): Re-prune Iceberg scans after partition
compensation #69678 (#70094)
picked from master #69678
Co-authored-by: daidai <[email protected]>
---
.../rules/rewrite/PruneFileScanPartition.java | 14 ++++++---
.../nereids/trees/copier/DeepCopierContext.java | 2 +-
.../trees/copier/LogicalPlanDeepCopier.java | 9 ++++--
.../trees/plans/logical/LogicalFileScan.java | 21 +++++++++++++
.../trees/copier/LogicalPlanDeepCopierTest.java | 35 ++++++++++++++++++++++
regression-test/data/mtmv_p0/test_iceberg_mtmv.out | 6 +++-
.../suites/mtmv_p0/test_iceberg_mtmv.groovy | 15 ++++++++++
7 files changed, 94 insertions(+), 8 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
index 3022fc7d7e8..f39f44378ed 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java
@@ -94,8 +94,12 @@ public class PruneFileScanPartition extends
OneRewriteRuleFactory {
Optional<SortedPartitionRanges<String>> sortedPartitionRanges =
Optional.empty();
boolean enableBinarySearch = ctx.getConnectContext() == null
||
ctx.getConnectContext().getSessionVariable().enableBinarySearchFilteringPartitions;
- if (enableBinarySearch) {
- sortedPartitionRanges = (Optional)
externalTable.getSortedPartitionRanges(scan);
+ if (enableBinarySearch && !nameToPartitionItem.isEmpty()) {
+ if (nameToPartitionItem.size() <
scan.getSelectedPartitions().totalPartitionNum) {
+ sortedPartitionRanges =
Optional.ofNullable(SortedPartitionRanges.build(nameToPartitionItem));
+ } else {
+ sortedPartitionRanges = (Optional)
externalTable.getSortedPartitionRanges(scan);
+ }
}
PartitionPruneResult<String> result = PartitionPruner.pruneWithResult(
partitionSlots, filter.getPredicate(), nameToPartitionItem,
ctx,
@@ -105,8 +109,10 @@ public class PruneFileScanPartition extends
OneRewriteRuleFactory {
for (String name : prunedPartitions) {
selectedPartitionItems.put(name, nameToPartitionItem.get(name));
}
- return new SelectedPartitions(nameToPartitionItem.size(),
selectedPartitionItems, true,
- result.hasPartitionPredicate);
+ boolean hasPartitionPredicate = result.hasPartitionPredicate
+ || scan.getSelectedPartitions().hasPartitionPredicate;
+ return new
SelectedPartitions(scan.getSelectedPartitions().totalPartitionNum,
selectedPartitionItems, true,
+ hasPartitionPredicate);
}
static List<Column> getPartitionColumnsForScan(ExternalTable
externalTable, LogicalFileScan scan) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/DeepCopierContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/DeepCopierContext.java
index b898a08f7f1..5bce27558c9 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/DeepCopierContext.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/DeepCopierContext.java
@@ -39,7 +39,7 @@ public class DeepCopierContext {
* to ensure they keep same after deep copy
*/
private final Map<RelationId, LogicalRelation> relationReplaceMap =
Maps.newHashMap();
- /** Whether copied OLAP scans should be marked for partition pruning
again. */
+ /** Whether copied catalog scans should be marked for partition pruning
again. */
private boolean invalidatePartitionPruning = false;
public void setInvalidatePartitionPruning(boolean
invalidatePartitionPruning) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
index 8144847d3ff..2c51dacfecc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java
@@ -39,6 +39,7 @@ import
org.apache.doris.nereids.trees.plans.logical.LogicalCTEProducer;
import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation;
import org.apache.doris.nereids.trees.plans.logical.LogicalExcept;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate;
import org.apache.doris.nereids.trees.plans.logical.LogicalHaving;
@@ -107,8 +108,12 @@ public class LogicalPlanDeepCopier extends
DefaultPlanRewriter<DeepCopierContext
}
LogicalCatalogRelation newRelation =
catalogRelation.withRelationId(StatementScopeIdGenerator.newRelationId());
- if (context.shouldInvalidatePartitionPruning() && newRelation
instanceof LogicalOlapScan) {
- newRelation = ((LogicalOlapScan)
newRelation).withPartitionPruned(false);
+ if (context.shouldInvalidatePartitionPruning()) {
+ if (newRelation instanceof LogicalOlapScan) {
+ newRelation = ((LogicalOlapScan)
newRelation).withPartitionPruned(false);
+ } else if (newRelation instanceof LogicalFileScan) {
+ newRelation = ((LogicalFileScan)
newRelation).withPartitionPruned(false);
+ }
}
updateReplaceMapWithOutput(catalogRelation, newRelation,
context.exprIdReplaceMap);
if (!catalogRelation.getVirtualColumns().isEmpty()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
index d19be98b97e..9c898bcc0fa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java
@@ -272,6 +272,16 @@ public class LogicalFileScan extends
LogicalCatalogRelation implements SupportPr
cachedOutputs, relationSchema, relationSnapshot, tableAlias);
}
+ /**
+ * Return a new scan with the specified partition pruning state.
+ */
+ public LogicalFileScan withPartitionPruned(boolean isPruned) {
+ if (selectedPartitions.isPruned == isPruned) {
+ return this;
+ }
+ return withSelectedPartitions(selectedPartitions.withPruned(isPruned));
+ }
+
@Override
public LogicalFileScan withRelationId(RelationId relationId) {
return new LogicalFileScan(relationId, (ExternalTable) table,
qualifier,
@@ -459,6 +469,17 @@ public class LogicalFileScan extends
LogicalCatalogRelation implements SupportPr
this.hasPartitionPredicate = hasPartitionPredicate;
}
+ /**
+ * Return a copy with the specified partition pruning state.
+ */
+ public SelectedPartitions withPruned(boolean isPruned) {
+ if (this.isPruned == isPruned) {
+ return this;
+ }
+ return new SelectedPartitions(totalPartitionNum,
selectedPartitions, isPruned,
+ hasPartitionPredicate);
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java
index 572258f4b45..96ae32db545 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopierTest.java
@@ -17,6 +17,8 @@
package org.apache.doris.nereids.trees.copier;
+import org.apache.doris.catalog.PartitionItem;
+import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.nereids.properties.OrderKey;
import org.apache.doris.nereids.trees.expressions.Add;
import org.apache.doris.nereids.trees.expressions.Alias;
@@ -29,16 +31,23 @@ import
org.apache.doris.nereids.trees.expressions.functions.scalar.Score;
import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.algebra.Repeat.RepeatType;
import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan;
+import
org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions;
import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalRepeat;
import org.apache.doris.nereids.types.BigIntType;
+import org.apache.doris.nereids.types.IntegerType;
import org.apache.doris.nereids.util.PlanConstructor;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableMap;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
+import org.mockito.Mockito;
+import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.stream.Collectors;
@@ -203,4 +212,30 @@ public class LogicalPlanDeepCopierTest {
Assertions.assertTrue(copiedPlan.hasPartitionPredicate());
}
+ @Test
+ public void testDeepCopyFileScanInvalidatesPartitionPruning() {
+ ExternalTable table = Mockito.mock(ExternalTable.class);
+ Map<String, PartitionItem> partitionItems = ImmutableMap.of(
+ "p1", Mockito.mock(PartitionItem.class));
+ SelectedPartitions selectedPartitions = new SelectedPartitions(2,
partitionItems, true, true);
+
Mockito.when(table.initSelectedPartitions(Mockito.any())).thenReturn(selectedPartitions);
+ Slot output = new SlotReference("p1", IntegerType.INSTANCE, true);
+ LogicalFileScan relationPlan = new
LogicalFileScan(PlanConstructor.getNextRelationId(), table,
+ ImmutableList.of("db"), Collections.emptyList(),
Optional.empty(), Optional.empty(),
+ Optional.empty(), Optional.of(ImmutableList.<Slot>of(output)));
+ DeepCopierContext context = new DeepCopierContext();
+ context.setInvalidatePartitionPruning(true);
+
+ LogicalFileScan copiedPlan =
+ (LogicalFileScan)
relationPlan.accept(LogicalPlanDeepCopier.INSTANCE, context);
+
+ Assertions.assertTrue(relationPlan.getSelectedPartitions().isPruned);
+ Assertions.assertFalse(copiedPlan.getSelectedPartitions().isPruned);
+ Assertions.assertTrue(copiedPlan.hasPartitionPredicate());
+
Assertions.assertEquals(relationPlan.getSelectedPartitions().totalPartitionNum,
+ copiedPlan.getSelectedPartitions().totalPartitionNum);
+
Assertions.assertEquals(relationPlan.getSelectedPartitions().selectedPartitions.keySet(),
+
copiedPlan.getSelectedPartitions().selectedPartitions.keySet());
+ }
+
}
diff --git a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
index 666a587c990..99373f8fd56 100644
--- a/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
+++ b/regression-test/data/mtmv_p0/test_iceberg_mtmv.out
@@ -129,6 +129,11 @@
2024-01-01T00:00 4
2024-01-02T00:00 3
+-- !new_source_partition_rewrite --
+2024-01-01T00:00 4
+2024-01-02T00:00 3
+2024-01-03T00:00 2
+
-- !evolution2 --
{"ts_month":null, "ts_day":"2024-12-03"}
{"ts_month":657, "ts_day":null}
@@ -176,4 +181,3 @@
-- !test6 --
{"create_date1_month":647, "create_date2_year":null}
{"create_date1_month":647, "create_date2_year":53}
-
diff --git a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
index b73b3c0b5eb..13767b6d255 100644
--- a/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
+++ b/regression-test/suites/mtmv_p0/test_iceberg_mtmv.groovy
@@ -263,6 +263,21 @@ suite("test_iceberg_mtmv",
"p0,external,iceberg,external_docker,external_docker_
order_qt_refresh_all_partition_rewrite "${mvSql}"
mv_rewrite_success("${mvSql}", "${mvName}")
+ // A new source partition must use partition compensation even when
the MV was previously complete.
+ sql """
+ insert into ${catalog_name}.${icebergDb}.${icebergTable3}
+ values (1, 3, "2024-01-03"), (2, 3, "2024-01-03")
+ """
+ sql """refresh table ${catalog_name}.${icebergDb}.${icebergTable3}"""
+ def showPartitionsAfterAdd = sql """show partitions from ${mvName}"""
+
assertFalse(showPartitionsAfterAdd.toString().contains("p_20240103000000_20240104000000"))
+ def explainNewPartition = sql """explain ${mvSql}"""
+ logger.info("explainNewPartition: " + explainNewPartition.toString())
+ assertTrue(explainNewPartition.toString().contains("VUNION"))
+ assertFalse(explainNewPartition.toString().contains("group expression
count exceeds"))
+ order_qt_new_source_partition_rewrite "${mvSql}"
+ mv_rewrite_success("${mvSql}", "${mvName}")
+
sql """drop materialized view if exists ${mvName};"""
sql """drop table if exists
${catalog_name}.${icebergDb}.${icebergTable3}"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
