This is an automated email from the ASF dual-hosted git repository. morrysnow pushed a commit to branch vector-index-dev in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/vector-index-dev by this push: new f5491c9fbd1 [opt](Nereids) push down virtual column into scan (#50521) f5491c9fbd1 is described below commit f5491c9fbd1ddf4cbbc6f400eb25eec75943cc00 Author: morrySnow <zhangwen...@selectdb.com> AuthorDate: Mon Apr 28 21:31:11 2025 +0800 [opt](Nereids) push down virtual column into scan (#50521) --- .../org/apache/doris/analysis/SlotDescriptor.java | 16 ++- .../java/org/apache/doris/analysis/SlotRef.java | 1 + .../glue/translator/PhysicalPlanTranslator.java | 22 ++++ .../glue/translator/PlanTranslatorContext.java | 6 + .../doris/nereids/jobs/executor/Rewriter.java | 2 + .../post/CommonSubExpressionCollector.java | 3 +- .../nereids/processor/post/PlanPostProcessors.java | 8 +- .../org/apache/doris/nereids/rules/RuleType.java | 1 + .../LogicalOlapScanToPhysicalOlapScan.java | 6 +- .../rewrite/PushDownVirualColumnsIntoOlapScan.java | 100 +++++++++++++++++ .../trees/plans/logical/LogicalOlapScan.java | 121 +++++++++++++-------- .../trees/plans/physical/PhysicalOlapScan.java | 31 ++++-- .../translator/PhysicalPlanTranslatorTest.java | 2 +- .../postprocess/MergeProjectPostProcessTest.java | 2 +- .../PushDownFilterThroughProjectTest.java | 4 +- .../doris/nereids/trees/plans/PlanEqualsTest.java | 6 +- 16 files changed, 263 insertions(+), 68 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 80076670746..5af4b378274 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -74,6 +74,7 @@ public class SlotDescriptor { // materialize them.Used to optimize to read less data and less memory usage private boolean needMaterialize = true; private boolean isAutoInc = false; + private Expr virtualColumn = null; public SlotDescriptor(SlotId id, TupleDescriptor parent) { this.id = id; @@ -268,6 +269,14 @@ public class SlotDescriptor { return column.getUniqueId(); } + public Expr getVirtualColumn() { + return virtualColumn; + } + + public void setVirtualColumn(Expr virtualColumn) { + this.virtualColumn = virtualColumn; + } + /** * Initializes a slot by setting its source expression information */ @@ -322,6 +331,9 @@ public class SlotDescriptor { if (subColPath != null) { tSlotDescriptor.setColumnPaths(subColPath); } + if (virtualColumn != null) { + tSlotDescriptor.setVirtualColumnExpr(virtualColumn.treeToThrift()); + } return tSlotDescriptor; } @@ -332,7 +344,8 @@ public class SlotDescriptor { return MoreObjects.toStringHelper(this).add("id", id.asInt()).add("parent", parentTupleId).add("col", colStr) .add("type", typeStr).add("materialized", isMaterialized).add("byteSize", byteSize) .add("byteOffset", byteOffset).add("slotIdx", slotIdx).add("nullable", getIsNullable()) - .add("isAutoIncrement", isAutoInc).add("subColPath", subColPath).toString(); + .add("isAutoIncrement", isAutoInc).add("subColPath", subColPath) + .add("virtualColumn", virtualColumn.toSql()).toString(); } @Override @@ -350,6 +363,7 @@ public class SlotDescriptor { .append(", nullable=").append(isNullable) .append(", isAutoIncrement=").append(isAutoInc) .append(", subColPath=").append(subColPath) + .append(", virtualColumn=").append(virtualColumn) .append("}") .toString(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java index 7078c90f1ed..80d756ca389 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotRef.java @@ -350,6 +350,7 @@ public class SlotRef extends Expr { msg.node_type = TExprNodeType.SLOT_REF; msg.slot_ref = new TSlotRef(desc.getId().asInt(), desc.getParent().getId().asInt()); msg.slot_ref.setColUniqueId(desc.getUniqueId()); + msg.slot_ref.setIsVirtualSlot(desc.getVirtualColumn() != null); msg.setLabel(label); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 86b4d025b9b..2ed49d2fa83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -760,6 +760,18 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla OlapTable olapTable = olapScan.getTable(); // generate real output tuple TupleDescriptor tupleDescriptor = generateTupleDesc(slots, olapTable, context); + + // put virtual column expr into slot desc + Map<ExprId, Expression> slotToVirtualColumnMap = olapScan.getSlotToVirtualColumnMap(); + for (SlotDescriptor slotDescriptor : tupleDescriptor.getSlots()) { + ExprId exprId = context.findExprId(slotDescriptor.getId()); + if (slotToVirtualColumnMap.containsKey(exprId)) { + slotDescriptor.setVirtualColumn(ExpressionTranslator.translate( + slotToVirtualColumnMap.get(exprId), context)); + context.getVirtualColumnIds().add(slotDescriptor.getId()); + } + } + // generate base index tuple because this fragment partitioned expr relay on slots of based index if (olapScan.getSelectedIndexId() != olapScan.getTable().getBaseIndexId()) { generateTupleDesc(olapScan.getBaseOutputs(), olapTable, context); @@ -2554,6 +2566,16 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla return tupleDescriptor; } + private TupleDescriptor generateTupleDescWithVirtualColumns(List<Slot> slotList, List<Expression> virtualColumns, + TableIf table, PlanTranslatorContext context) { + TupleDescriptor tupleDescriptor = context.generateTupleDesc(); + tupleDescriptor.setTable(table); + for (Slot slot : slotList) { + context.createSlotDesc(tupleDescriptor, (SlotReference) slot, table); + } + return tupleDescriptor; + } + private PlanFragment connectJoinNode(HashJoinNode hashJoinNode, PlanFragment leftFragment, PlanFragment rightFragment, PlanTranslatorContext context, AbstractPlan join) { hashJoinNode.setChild(0, leftFragment.getPlanRoot()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java index f40a4e2e8f4..05a0ad5199f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PlanTranslatorContext.java @@ -114,6 +114,8 @@ public class PlanTranslatorContext { private final Map<ScanNode, Set<SlotId>> statsUnknownColumnsMap = Maps.newHashMap(); + private final Set<SlotId> virtualColumnIds = Sets.newHashSet(); + public PlanTranslatorContext(CascadesContext ctx) { this.connectContext = ctx.getConnectContext(); this.translator = new RuntimeFilterTranslator(ctx.getRuntimeFilterContext()); @@ -337,4 +339,8 @@ public class PlanTranslatorContext { public TPushAggOp getRelationPushAggOp(RelationId relationId) { return tablePushAggOp.getOrDefault(relationId, TPushAggOp.NONE); } + + public Set<SlotId> getVirtualColumnIds() { + return virtualColumnIds; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 80c8760d0e5..0f9cd53c6af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -132,6 +132,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughUnion; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughUnion; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughWindow; +import org.apache.doris.nereids.rules.rewrite.PushDownVirualColumnsIntoOlapScan; import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin; import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation; import org.apache.doris.nereids.rules.rewrite.PushProjectIntoUnion; @@ -460,6 +461,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown(new SumLiteralRewrite(), new MergePercentileToArray()) ), + topDown(new PushDownVirualColumnsIntoOlapScan()), topic("Push project and filter on cte consumer to cte producer", topDown( new CollectFilterAboveConsumer(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java index 520902c0439..29c70b7ed3f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/CommonSubExpressionCollector.java @@ -61,8 +61,7 @@ public class CommonSubExpressionCollector extends ExpressionVisitor<Integer, Boo // ArrayItemSlot and ArrayItemReference could not be common expressions // TODO: could not extract common expression when expression contains same lambda expression // because ArrayItemSlot in Lambda are not same. - if (expressions.contains(expr) - && !(inLambda && expr.containsType(ArrayItemSlot.class, ArrayItemReference.class))) { + if (!(inLambda && expr.containsType(ArrayItemSlot.class, ArrayItemReference.class))) { Set<Expression> commonExpression = getExpressionsFromDepthMap(depth, commonExprByDepth); commonExpression.add(expr); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java index a8654e27291..86801dc1b26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/processor/post/PlanPostProcessors.java @@ -62,10 +62,10 @@ public class PlanPostProcessors { builder.add(new RemoveUselessProjectPostProcessor()); builder.add(new MergeProjectPostProcessor()); builder.add(new RecomputeLogicalPropertiesProcessor()); - if (cascadesContext.getConnectContext().getSessionVariable().enableAggregateCse) { - builder.add(new ProjectAggregateExpressionsForCse()); - } - builder.add(new CommonSubExpressionOpt()); + // if (cascadesContext.getConnectContext().getSessionVariable().enableAggregateCse) { + // builder.add(new ProjectAggregateExpressionsForCse()); + // } + // builder.add(new CommonSubExpressionOpt()); // DO NOT replace PLAN NODE from here if (cascadesContext.getConnectContext().getSessionVariable().pushTopnToAgg) { builder.add(new PushTopnToAgg()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 9cee3077f13..5173fbc3d50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -316,6 +316,7 @@ public enum RuleType { PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE), PUSH_CONJUNCTS_INTO_ODBC_SCAN(RuleTypeClass.REWRITE), PUSH_CONJUNCTS_INTO_ES_SCAN(RuleTypeClass.REWRITE), + PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN(RuleTypeClass.REWRITE), OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE), PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE), PUSH_COUNT_INTO_UNION_ALL(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 4d006e557ac..eb5e00a32ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -62,7 +62,8 @@ public class LogicalOlapScanToPhysicalOlapScan extends OneImplementationRuleFact Optional.empty(), olapScan.getLogicalProperties(), olapScan.getTableSample(), - olapScan.getOperativeSlots()) + olapScan.getOperativeSlots(), + olapScan.getVirtualColumns()) ).toRule(RuleType.LOGICAL_OLAP_SCAN_TO_PHYSICAL_OLAP_SCAN_RULE); } @@ -115,7 +116,8 @@ public class LogicalOlapScanToPhysicalOlapScan extends OneImplementationRuleFact // If the length of the column in the bucket key changes after DDL, the length cannot be // determined. As a result, some bucket fields are lost in the query execution plan. // So here we use the column name to avoid this problem - if (((SlotReference) slot).getColumn().get().getName().equalsIgnoreCase(column.getName())) { + if (((SlotReference) slot).getColumn().isPresent() && ((SlotReference) slot).getColumn().get() + .getName().equalsIgnoreCase(column.getName())) { hashColumns.add(slot.getExprId()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirualColumnsIntoOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirualColumnsIntoOlapScan.java new file mode 100644 index 00000000000..d2c3bcbb5d3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVirualColumnsIntoOlapScan.java @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.processor.post.CommonSubExpressionCollector; +import org.apache.doris.nereids.processor.post.CommonSubExpressionOpt.ExpressionReplacer; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.WhenClause; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * extract virtual column from filter and push down them into olap scan. + */ +public class PushDownVirualColumnsIntoOlapScan extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalProject(logicalFilter(logicalOlapScan() + .when(s -> s.getVirtualColumns().isEmpty()))) + .then(project -> { + // 1. extract filter common expr + // 2. generate virtual column from common expr and add them to scan + // 3. replace filter + // 4. replace project + LogicalFilter<LogicalOlapScan> filter = project.child(); + LogicalOlapScan logicalOlapScan = filter.child(); + CommonSubExpressionCollector collector = new CommonSubExpressionCollector(); + for (Expression expr : filter.getConjuncts()) { + collector.collect(expr); + } + Map<Expression, Alias> aliasMap = new LinkedHashMap<>(); + if (!collector.commonExprByDepth.isEmpty()) { + for (int i = 1; i <= collector.commonExprByDepth.size(); i++) { + Set<Expression> exprsInDepth = CommonSubExpressionCollector + .getExpressionsFromDepthMap(i, collector.commonExprByDepth); + exprsInDepth.forEach(expr -> { + if (!(expr instanceof WhenClause)) { + // case whenClause1 whenClause2 END + // whenClause should not be regarded as common-sub-expression, because + // cse will be replaced by a slot, after rewrite the case clause becomes: + // 'case slot whenClause2 END' + // This is illegal. + Expression rewritten = expr.accept(ExpressionReplacer.INSTANCE, aliasMap); + // if rewritten is already alias, use it directly, + // because in materialized view rewriting + // Should keep out slot immutably after rewritten successfully + aliasMap.put(expr, rewritten instanceof Alias + ? (Alias) rewritten : new Alias(rewritten)); + } + }); + } + } + List<NamedExpression> virtualColumns = Lists.newArrayList(); + Map<Expression, Slot> replaceMap = Maps.newHashMap(); + for (Map.Entry<Expression, Alias> entry : aliasMap.entrySet()) { + Alias alias = entry.getValue(); + replaceMap.put(entry.getKey(), alias.toSlot()); + virtualColumns.add(alias); + } + logicalOlapScan = logicalOlapScan.withVirtualColumns(virtualColumns); + Set<Expression> conjuncts = ExpressionUtils.replace(filter.getConjuncts(), replaceMap); + List<NamedExpression> projections = ExpressionUtils.replace( + (List) project.getProjects(), replaceMap); + LogicalFilter<?> newFilter = filter.withConjunctsAndChild(conjuncts, logicalOlapScan); + LogicalProject<?> newProject = project.withProjectsAndChild(projections, newFilter); + return newProject; + }).toRule(RuleType.PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index ce57f026ea5..d28250b394e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -29,6 +29,7 @@ import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule; import org.apache.doris.nereids.trees.TableSample; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; @@ -63,6 +64,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; /** * Logical OlapScan. @@ -138,6 +140,9 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan private final Map<String, Set<List<String>>> colToSubPathsMap; private final Map<Slot, Map<List<String>, SlotReference>> subPathToSlotMap; + // use for virtual slot + private final List<NamedExpression> virtualColumns; + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } @@ -151,7 +156,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan ImmutableList.of(), -1, false, PreAggStatus.unset(), ImmutableList.of(), ImmutableList.of(), Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of(), - ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds, @@ -159,7 +164,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, tabletIds, -1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots); + tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots, ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions, @@ -168,7 +173,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, tabletIds, -1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots); + tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots, ImmutableList.of()); } public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds, @@ -179,7 +184,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, false, tabletIds, selectedIndexId, true, preAggStatus, specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of(), - ImmutableList.of(), operativeSlots); + ImmutableList.of(), operativeSlots, ImmutableList.of()); } /** @@ -193,7 +198,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan List<String> hints, Map<Pair<Long, String>, Slot> cacheSlotWithSlotName, Optional<TableSample> tableSample, boolean directMvScan, Map<String, Set<List<String>>> colToSubPathsMap, List<Long> specifiedTabletIds, - Collection<Slot> operativeSlots) { + Collection<Slot> operativeSlots, List<NamedExpression> virtualColumns) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties, operativeSlots); Preconditions.checkArgument(selectedPartitionIds != null, @@ -225,6 +230,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this.directMvScan = directMvScan; this.colToSubPathsMap = colToSubPathsMap; this.subPathToSlotMap = Maps.newHashMap(); + this.virtualColumns = Utils.fastToImmutableList(virtualColumns); } public List<Long> getSelectedPartitionIds() { @@ -244,39 +250,11 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan "indexName", getSelectedMaterializedIndexName().orElse("<index_not_selected>"), "selectedIndexId", selectedIndexId, "preAgg", preAggStatus, - "operativeCol", operativeSlots + "operativeCol", operativeSlots, + "virtualColumns", virtualColumns ); } - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if (!super.equals(o)) { - return false; - } - LogicalOlapScan that = (LogicalOlapScan) o; - return selectedIndexId == that.selectedIndexId && indexSelected == that.indexSelected - && partitionPruned == that.partitionPruned && Objects.equals(preAggStatus, that.preAggStatus) - && Objects.equals(selectedTabletIds, that.selectedTabletIds) - && Objects.equals(manuallySpecifiedPartitions, that.manuallySpecifiedPartitions) - && Objects.equals(manuallySpecifiedTabletIds, that.manuallySpecifiedTabletIds) - && Objects.equals(selectedPartitionIds, that.selectedPartitionIds) - && Objects.equals(hints, that.hints) - && Objects.equals(tableSample, that.tableSample); - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), selectedIndexId, indexSelected, preAggStatus, cacheSlotWithSlotName, - selectedTabletIds, partitionPruned, manuallySpecifiedTabletIds, manuallySpecifiedPartitions, - selectedPartitionIds, hints, tableSample); - } - @Override public LogicalOlapScan withGroupExpression(Optional<GroupExpression> groupExpression) { return new LogicalOlapScan(relationId, (Table) table, qualifier, @@ -284,7 +262,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } @Override @@ -294,7 +272,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } /** @@ -306,15 +284,21 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } + /** + * with sync materialized index id. + * @param indexId materialized index id for scan + * @return scan with materialized index id + */ public LogicalOlapScan withMaterializedIndexSelected(long indexId) { return new LogicalOlapScan(relationId, (Table) table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, - tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, operativeSlots); + tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, + operativeSlots, virtualColumns); } /** @@ -326,7 +310,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } /** @@ -338,7 +322,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } /** @@ -350,7 +334,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } /** @@ -362,7 +346,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } @Override @@ -373,7 +357,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, false, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap, selectedTabletIds, - operativeSlots); + operativeSlots, virtualColumns); } @Override @@ -445,6 +429,10 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan } } } + // add virtual slots + for (NamedExpression virtualColumn : virtualColumns) { + slots.add(virtualColumn.toSlot()); + } return slots.build(); } @@ -462,6 +450,10 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan slots.addAll(generateUniqueSlot( olapTable, c, indexId == ((OlapTable) table).getBaseIndexId(), indexId)); } + // add virtual slots, TODO: maybe wrong, should test virtual column + sync mv + for (NamedExpression virtualColumn : virtualColumns) { + slots.add(virtualColumn.toSlot()); + } return slots; } @@ -508,6 +500,10 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan return preAggStatus.isUnset(); } + public List<NamedExpression> getVirtualColumns() { + return virtualColumns; + } + private List<SlotReference> createSlotsVectorized(List<Column> columns) { List<String> qualified = qualified(); SlotReference[] slots = new SlotReference[columns.size()]; @@ -667,7 +663,25 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, - manuallySpecifiedTabletIds, operativeSlots); + manuallySpecifiedTabletIds, operativeSlots, virtualColumns); + } + + /** + * add virtual column to olap scan. + * @param virtualColumns generated virtual columns + * @return scan with virtual columns + */ + public LogicalOlapScan withVirtualColumns(List<NamedExpression> virtualColumns) { + LogicalProperties logicalProperties = getLogicalProperties(); + List<Slot> output = Lists.newArrayList(logicalProperties.getOutput()); + output.addAll(virtualColumns.stream().map(NamedExpression::toSlot).collect(Collectors.toList())); + logicalProperties = new LogicalProperties(() -> output, this::computeDataTrait); + return new LogicalOlapScan(relationId, (Table) table, qualifier, + groupExpression, Optional.of(logicalProperties), + selectedPartitionIds, partitionPruned, selectedTabletIds, + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, + manuallySpecifiedTabletIds, operativeSlots, virtualColumns); } Map<Slot, Slot> constructReplaceMap(MTMV mtmv) { @@ -703,4 +717,21 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan public List<Slot> getOperativeSlots() { return operativeSlots; } + + @Override + public boolean equals(Object o) { + if (o == null || getClass() != o.getClass()) { + return false; + } + if (!super.equals(o)) { + return false; + } + LogicalOlapScan that = (LogicalOlapScan) o; + return Objects.equals(virtualColumns, that.virtualColumns); + } + + @Override + public int hashCode() { + return super.hashCode(); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 74c5cb0be3b..0092abee62f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -23,6 +23,9 @@ import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; @@ -39,8 +42,10 @@ import org.json.JSONObject; import java.util.Collection; import java.util.List; +import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.stream.Collectors; /** * Physical olap scan plan. @@ -56,6 +61,9 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca private final Optional<TableSample> tableSample; private final ImmutableList<Slot> operativeSlots; + // use for virtual slot + private final List<NamedExpression> virtualColumns; + /** * Constructor for PhysicalOlapScan. */ @@ -63,12 +71,12 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca List<Long> selectedTabletIds, List<Long> selectedPartitionIds, DistributionSpec distributionSpec, PreAggStatus preAggStatus, List<Slot> baseOutputs, Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, - Optional<TableSample> tableSample, List<Slot> operativeSlots) { + Optional<TableSample> tableSample, List<Slot> operativeSlots, List<NamedExpression> virtualColumns) { this(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, logicalProperties, null, - null, tableSample, operativeSlots); + null, tableSample, operativeSlots, virtualColumns); } /** @@ -80,7 +88,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, Optional<TableSample> tableSample, - Collection<Slot> operativeSlots) { + Collection<Slot> operativeSlots, List<NamedExpression> virtualColumns) { super(id, PlanType.PHYSICAL_OLAP_SCAN, olapTable, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); this.selectedIndexId = selectedIndexId; @@ -91,6 +99,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca this.baseOutputs = ImmutableList.copyOf(baseOutputs); this.tableSample = tableSample; this.operativeSlots = ImmutableList.copyOf(operativeSlots); + this.virtualColumns = ImmutableList.copyOf(virtualColumns); } @Override @@ -124,6 +133,14 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca return baseOutputs; } + public List<NamedExpression> getVirtualColumns() { + return virtualColumns; + } + + public Map<ExprId, Expression> getSlotToVirtualColumnMap() { + return virtualColumns.stream().collect(Collectors.toMap(e -> e.toSlot().getExprId(), e -> e)); + } + @Override public String toString() { StringBuilder builder = new StringBuilder(); @@ -184,7 +201,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca public PhysicalOlapScan withGroupExpression(Optional<GroupExpression> groupExpression) { return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, - groupExpression, getLogicalProperties(), tableSample, operativeSlots); + groupExpression, getLogicalProperties(), tableSample, operativeSlots, virtualColumns); } @Override @@ -192,7 +209,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca Optional<LogicalProperties> logicalProperties, List<Plan> children) { return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, - logicalProperties.get(), tableSample, operativeSlots); + logicalProperties.get(), tableSample, operativeSlots, virtualColumns); } @Override @@ -200,7 +217,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, - getLogicalProperties(), physicalProperties, statistics, tableSample, operativeSlots); + getLogicalProperties(), physicalProperties, statistics, tableSample, operativeSlots, virtualColumns); } @Override @@ -226,6 +243,6 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca return new PhysicalOlapScan(relationId, (OlapTable) table, qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), getPhysicalProperties(), statistics, - tableSample, operativeSlots); + tableSample, operativeSlots, virtualColumns); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index 5fc63eea3bb..86a0b366b8e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -68,7 +68,7 @@ public class PhysicalPlanTranslatorTest { PhysicalOlapScan scan = new PhysicalOlapScan(StatementScopeIdGenerator.newRelationId(), t1, qualifier, t1.getBaseIndexId(), Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, Optional.empty(), - ImmutableList.of()); + ImmutableList.of(), ImmutableList.of()); Literal t1FilterRight = new IntegerLiteral(1); Expression t1FilterExpr = new GreaterThan(col1, t1FilterRight); PhysicalFilter<PhysicalOlapScan> filter = diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java index c4648ed5a6b..8237b38209e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java @@ -78,7 +78,7 @@ public class MergeProjectPostProcessTest { LogicalProperties t1Properties = new LogicalProperties(() -> t1Output, () -> DataTrait.EMPTY_TRAIT); PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), - Optional.empty(), t1Properties, Optional.empty(), ImmutableList.of()); + Optional.empty(), t1Properties, Optional.empty(), ImmutableList.of(), ImmutableList.of()); Alias x = new Alias(a, "x"); List<NamedExpression> projList3 = Lists.newArrayList(x, b, c); PhysicalProject proj3 = new PhysicalProject(projList3, placeHolder, scan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java index d93b1111c19..ded93cec2dd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java @@ -93,7 +93,7 @@ public class PushDownFilterThroughProjectTest { PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, - Optional.empty(), ImmutableList.of()); + Optional.empty(), ImmutableList.of(), ImmutableList.of()); Alias x = new Alias(a, "x"); List<NamedExpression> projList3 = Lists.newArrayList(x, b, c); PhysicalProject proj3 = new PhysicalProject(projList3, placeHolder, scan); @@ -132,7 +132,7 @@ public class PushDownFilterThroughProjectTest { PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, - Optional.empty(), ImmutableList.of()); + Optional.empty(), ImmutableList.of(), ImmutableList.of()); Alias x = new Alias(a, "x"); List<NamedExpression> projList3 = Lists.newArrayList(x, b, c); PhysicalProject proj3 = new PhysicalProject(projList3, placeHolder, scan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java index 062bd3e71b9..5b8cad01c05 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java @@ -268,20 +268,20 @@ class PlanEqualsTest { 1L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties, Optional.empty(), - ImmutableList.of()); + ImmutableList.of(), ImmutableList.of()); PhysicalOlapScan expected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("a"), 1L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties, Optional.empty(), - ImmutableList.of()); + ImmutableList.of(), ImmutableList.of()); Assertions.assertEquals(expected, actual); PhysicalOlapScan unexpected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("b"), 12345L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties, Optional.empty(), - ImmutableList.of()); + ImmutableList.of(), ImmutableList.of()); Assertions.assertNotEquals(unexpected, actual); } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org