This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch vector-index-dev in repository https://gitbox.apache.org/repos/asf/doris.git
commit ee63976adb931c545939810a8f66c0d397c2a621 Author: morrySnow <zhangwen...@selectdb.com> AuthorDate: Thu May 8 17:15:06 2025 +0800 [feature](vector) push down ann topn into scan --- .../glue/translator/PhysicalPlanTranslator.java | 20 +++ .../doris/nereids/jobs/executor/Rewriter.java | 4 +- .../org/apache/doris/nereids/rules/RuleType.java | 1 + .../LogicalOlapScanToPhysicalOlapScan.java | 4 +- .../rewrite/PushDownVectorTopNIntoOlapScan.java | 138 +++++++++++++++++++++ .../trees/copier/LogicalPlanDeepCopier.java | 2 + .../expressions/functions/scalar/L2Distance.java | 4 +- .../trees/plans/logical/LogicalOlapScan.java | 118 ++++++++++++------ .../trees/plans/physical/PhysicalOlapScan.java | 50 ++++++-- .../org/apache/doris/planner/OlapScanNode.java | 32 +++++ .../translator/PhysicalPlanTranslatorTest.java | 2 +- .../postprocess/MergeProjectPostProcessTest.java | 3 +- .../PushDownFilterThroughProjectTest.java | 4 +- .../doris/nereids/trees/plans/PlanEqualsTest.java | 6 +- 14 files changed, 327 insertions(+), 61 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 9bfdf4830e6..8370d5ca7dc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -838,6 +838,26 @@ public class PhysicalPlanTranslator extends DefaultPlanVisitor<PlanFragment, Pla OlapScanNode olapScanNode = new OlapScanNode(context.nextPlanNodeId(), tupleDescriptor, "OlapScanNode"); olapScanNode.setNereidsId(olapScan.getId()); context.getNereidsIdToPlanNodeIdMap().put(olapScan.getId(), olapScanNode.getId()); + + // translate ann topn info + if (!olapScan.getAnnOrderKeys().isEmpty()) { + TupleDescriptor annSortTuple = olapScanNode.getTupleDesc(); + List<Expr> orderingExprs = Lists.newArrayList(); + List<Boolean> ascOrders = Lists.newArrayList(); + List<Boolean> nullsFirstParams = Lists.newArrayList(); + List<OrderKey> annOrderKeys = olapScan.getAnnOrderKeys(); + annOrderKeys.forEach(k -> { + orderingExprs.add(ExpressionTranslator.translate(k.getExpr(), context)); + ascOrders.add(k.isAsc()); + nullsFirstParams.add(k.isNullFirst()); + }); + SortInfo annSortInfo = new SortInfo(orderingExprs, ascOrders, nullsFirstParams, annSortTuple); + olapScanNode.setAnnSortInfo(annSortInfo); + } + if (olapScan.getAnnLimit().isPresent()) { + olapScanNode.setAnnSortLimit(olapScan.getAnnLimit().get()); + } + // TODO: move all node set cardinality into one place if (olapScan.getStats() != null) { // NOTICE: we should not set stats row count diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index ad7ab221d5b..a60eb0d2522 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -132,7 +132,7 @@ import org.apache.doris.nereids.rules.rewrite.PushDownTopNDistinctThroughUnion; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughUnion; import org.apache.doris.nereids.rules.rewrite.PushDownTopNThroughWindow; -import org.apache.doris.nereids.rules.rewrite.PushDownVirualColumnsIntoOlapScan; +import org.apache.doris.nereids.rules.rewrite.PushDownVectorTopNIntoOlapScan; import org.apache.doris.nereids.rules.rewrite.PushFilterInsideJoin; import org.apache.doris.nereids.rules.rewrite.PushProjectIntoOneRowRelation; import org.apache.doris.nereids.rules.rewrite.PushProjectIntoUnion; @@ -470,7 +470,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown(new CollectPredicateOnScan()) ), topDown(new PushDownVirualColumnsIntoOlapScan()), - + topDown(new PushDownVectorTopNIntoOlapScan()), topic("Push project and filter on cte consumer to cte producer", topDown( new CollectFilterAboveConsumer(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index e98bbf2abe3..e85de463a6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -314,6 +314,7 @@ public enum RuleType { PUSH_CONJUNCTS_INTO_ODBC_SCAN(RuleTypeClass.REWRITE), PUSH_CONJUNCTS_INTO_ES_SCAN(RuleTypeClass.REWRITE), PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN(RuleTypeClass.REWRITE), + PUSH_DOWN_VECTOR_TOPN_INTO_OLAP_SCAN(RuleTypeClass.REWRITE), OLAP_SCAN_TABLET_PRUNE(RuleTypeClass.REWRITE), PUSH_AGGREGATE_TO_OLAP_SCAN(RuleTypeClass.REWRITE), PUSH_COUNT_INTO_UNION_ALL(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java index 8b3a436d72d..dd59136b1ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOlapScanToPhysicalOlapScan.java @@ -63,7 +63,9 @@ public class LogicalOlapScanToPhysicalOlapScan extends OneImplementationRuleFact olapScan.getLogicalProperties(), olapScan.getTableSample(), olapScan.getOperativeSlots(), - olapScan.getVirtualColumns()) + olapScan.getVirtualColumns(), + olapScan.getAnnOrderKeys(), + olapScan.getAnnLimit()) ).toRule(RuleType.LOGICAL_OLAP_SCAN_TO_PHYSICAL_OLAP_SCAN_RULE); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoOlapScan.java new file mode 100644 index 00000000000..ac5dc07aab4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownVectorTopNIntoOlapScan.java @@ -0,0 +1,138 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.analysis.IndexDef.IndexType; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Index; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.L2Distance; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalTopN; +import org.apache.doris.nereids.util.ExpressionUtils; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Maps; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; + +/** + * extract virtual column from filter and push down them into olap scan. + */ +public class PushDownVectorTopNIntoOlapScan implements RewriteRuleFactory { + @Override + public List<Rule> buildRules() { + return ImmutableList.of( + logicalTopN(logicalProject(logicalOlapScan())).when(t -> t.getOrderKeys().size() == 1).then(topN -> { + LogicalProject<LogicalOlapScan> project = topN.child(); + LogicalOlapScan scan = project.child(); + return pushDown(topN, project, scan, Optional.empty()); + }).toRule(RuleType.PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN), + logicalTopN(logicalProject(logicalFilter(logicalOlapScan()))) + .when(t -> t.getOrderKeys().size() == 1).then(topN -> { + LogicalProject<LogicalFilter<LogicalOlapScan>> project = topN.child(); + LogicalFilter<LogicalOlapScan> filter = project.child(); + LogicalOlapScan scan = filter.child(); + return pushDown(topN, project, scan, Optional.of(filter)); + }).toRule(RuleType.PUSH_DOWN_VIRTUAL_COLUMNS_INTO_OLAP_SCAN) + ); + } + + private Plan pushDown( + LogicalTopN<?> topN, + LogicalProject<?> project, + LogicalOlapScan scan, + Optional<LogicalFilter<?>> optionalFilter) { + Expression orderKey = topN.getOrderKeys().get(0).getExpr(); + if (!(orderKey instanceof SlotReference)) { + return null; + } + SlotReference keySlot = (SlotReference) orderKey; + Expression orderKeyExpr = null; + Alias orderKeyAlias = null; + for (NamedExpression projection : project.getProjects()) { + if (projection.toSlot().equals(keySlot) && projection instanceof Alias) { + orderKeyExpr = ((Alias) projection).child(); + orderKeyAlias = (Alias) projection; + break; + } + } + if (orderKeyExpr == null) { + return null; + } + if (!(orderKeyExpr instanceof L2Distance)) { + return null; + } + L2Distance l2Distance = (L2Distance) orderKeyExpr; + Expression left = l2Distance.left(); + while (left instanceof Cast) { + left = ((Cast) left).child(); + } + if (!(left instanceof SlotReference && ((L2Distance) orderKeyExpr).right().isConstant())) { + return null; + } + SlotReference leftInput = (SlotReference) left; + if (!leftInput.getColumn().isPresent() || !leftInput.getTable().isPresent()) { + return null; + } + TableIf table = leftInput.getTable().get(); + Column column = leftInput.getColumn().get(); + boolean hasAnnIndexOnColumn = false; + for (Index index : table.getTableIndexes().getIndexes()) { + if (index.getIndexType() == IndexType.ANN) { + if (index.getColumns().size() != 1) { + continue; + } + if (index.getColumns().get(0).equalsIgnoreCase(column.getName())) { + hasAnnIndexOnColumn = true; + break; + } + } + } + if (!hasAnnIndexOnColumn) { + return null; + } + Plan plan = scan.withVirtualColumnsAndTopN(ImmutableList.of(orderKeyAlias), + topN.getOrderKeys(), topN.getLimit() + topN.getOffset()); + Map<Expression, Expression> replaceMap = Maps.newHashMap(); + replaceMap.put(orderKeyAlias, orderKeyAlias.toSlot()); + replaceMap.put(orderKeyExpr, orderKeyAlias.toSlot()); + if (optionalFilter.isPresent()) { + LogicalFilter<?> filter = optionalFilter.get(); + Set<Expression> newConjuncts = ExpressionUtils.replace(filter.getConjuncts(), replaceMap); + plan = filter.withConjunctsAndChild(newConjuncts, plan); + } + List<NamedExpression> newProjections = ExpressionUtils + .replaceNamedExpressions(project.getProjects(), replaceMap); + LogicalProject<?> newProject = project.withProjectsAndChild(newProjections, plan); + return topN.withChildren(newProject); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 2e231c00d22..a6e724f8454 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -98,6 +98,8 @@ public class LogicalPlanDeepCopier extends DefaultPlanRewriter<DeepCopierContext return newRelation; } + // TODO update scan + @Override public Plan visitLogicalCatalogRelation(LogicalCatalogRelation relation, DeepCopierContext context) { if (context.getRelationReplaceMap().containsKey(relation.getRelationId())) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java index 14ffee389ae..87440363680 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/scalar/L2Distance.java @@ -22,7 +22,7 @@ import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.AlwaysNullable; import org.apache.doris.nereids.trees.expressions.functions.ComputePrecisionForArrayItemAgg; import org.apache.doris.nereids.trees.expressions.functions.ExplicitlyCastableSignature; -import org.apache.doris.nereids.trees.expressions.shape.UnaryExpression; +import org.apache.doris.nereids.trees.expressions.shape.BinaryExpression; import org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitor; import org.apache.doris.nereids.types.ArrayType; import org.apache.doris.nereids.types.DoubleType; @@ -36,7 +36,7 @@ import java.util.List; * l2_distance function */ public class L2Distance extends ScalarFunction implements ExplicitlyCastableSignature, - ComputePrecisionForArrayItemAgg, UnaryExpression, AlwaysNullable { + ComputePrecisionForArrayItemAgg, BinaryExpression, AlwaysNullable { public static final List<FunctionSignature> SIGNATURES = ImmutableList.of( FunctionSignature.ret(DoubleType.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java index fae2e4bfba6..b1f57386aab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java @@ -26,6 +26,7 @@ import org.apache.doris.mtmv.MTMVCache; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.rules.rewrite.mv.AbstractSelectMaterializedIndexRule; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -142,6 +143,10 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan // use for virtual slot private final List<NamedExpression> virtualColumns; + // use for ann push down + private final List<OrderKey> annOrderKeys; + private final Optional<Long> annLimit; + public LogicalOlapScan(RelationId id, OlapTable table) { this(id, table, ImmutableList.of()); } @@ -155,7 +160,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan ImmutableList.of(), -1, false, PreAggStatus.unset(), ImmutableList.of(), ImmutableList.of(), Maps.newHashMap(), Optional.empty(), false, ImmutableMap.of(), - ImmutableList.of(), ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty()); } public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds, @@ -163,16 +168,21 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this(id, table, qualifier, Optional.empty(), Optional.empty(), table.getPartitionIds(), false, tabletIds, -1, false, PreAggStatus.unset(), ImmutableList.of(), hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots, ImmutableList.of()); + tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots, + ImmutableList.of(), ImmutableList.of(), Optional.empty()); } + /** + * constructor. + */ public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> specifiedPartitions, List<Long> tabletIds, List<String> hints, Optional<TableSample> tableSample, List<Slot> operativeSlots) { this(id, table, qualifier, Optional.empty(), Optional.empty(), // must use specifiedPartitions here for prune partition by sql like 'select * from t partition p1' specifiedPartitions, false, tabletIds, -1, false, PreAggStatus.unset(), specifiedPartitions, hints, Maps.newHashMap(), - tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots, ImmutableList.of()); + tableSample, false, ImmutableMap.of(), ImmutableList.of(), operativeSlots, + ImmutableList.of(), ImmutableList.of(), Optional.empty()); } public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier, List<Long> tabletIds, @@ -183,7 +193,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, false, tabletIds, selectedIndexId, true, preAggStatus, specifiedPartitions, hints, Maps.newHashMap(), tableSample, true, ImmutableMap.of(), - ImmutableList.of(), operativeSlots, ImmutableList.of()); + ImmutableList.of(), operativeSlots, ImmutableList.of(), ImmutableList.of(), Optional.empty()); } /** @@ -197,7 +207,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan List<String> hints, Map<Pair<Long, String>, Slot> cacheSlotWithSlotName, Optional<TableSample> tableSample, boolean directMvScan, Map<String, Set<List<String>>> colToSubPathsMap, List<Long> specifiedTabletIds, - Collection<Slot> operativeSlots, List<NamedExpression> virtualColumns) { + Collection<Slot> operativeSlots, List<NamedExpression> virtualColumns, + List<OrderKey> annOrderKeys, Optional<Long> annLimit) { super(id, PlanType.LOGICAL_OLAP_SCAN, table, qualifier, groupExpression, logicalProperties, operativeSlots); Preconditions.checkArgument(selectedPartitionIds != null, @@ -230,6 +241,8 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan this.colToSubPathsMap = colToSubPathsMap; this.subPathToSlotMap = Maps.newHashMap(); this.virtualColumns = Utils.fastToImmutableList(virtualColumns); + this.annOrderKeys = Utils.fastToImmutableList(annOrderKeys); + this.annLimit = annLimit; } public List<Long> getSelectedPartitionIds() { @@ -264,7 +277,9 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan "selectedIndexId", selectedIndexId, "preAgg", preAggStatus, "operativeCol", operativeSlots, - "virtualColumns", virtualColumns + "virtualColumns", virtualColumns, + "orderKeys", annOrderKeys, + "limit", annLimit ); } @@ -275,7 +290,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } @Override @@ -285,7 +300,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } /** @@ -297,7 +312,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, true, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } /** @@ -311,7 +326,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, indexId, true, PreAggStatus.unset(), manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } /** @@ -323,7 +338,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } /** @@ -335,7 +350,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } /** @@ -347,7 +362,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } /** @@ -359,7 +374,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, manuallySpecifiedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); } @Override @@ -370,7 +385,46 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, false, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, Maps.newHashMap(), tableSample, directMvScan, colToSubPathsMap, selectedTabletIds, - operativeSlots, virtualColumns); + operativeSlots, virtualColumns, annOrderKeys, annLimit); + } + + /** + * add virtual column to olap scan. + * @param virtualColumns generated virtual columns + * @return scan with virtual columns + */ + public LogicalOlapScan withVirtualColumns(List<NamedExpression> virtualColumns) { + LogicalProperties logicalProperties = getLogicalProperties(); + List<Slot> output = Lists.newArrayList(logicalProperties.getOutput()); + output.addAll(virtualColumns.stream().map(NamedExpression::toSlot).collect(Collectors.toList())); + logicalProperties = new LogicalProperties(() -> output, this::computeDataTrait); + return new LogicalOlapScan(relationId, (Table) table, qualifier, + groupExpression, Optional.of(logicalProperties), + selectedPartitionIds, partitionPruned, selectedTabletIds, + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, + manuallySpecifiedTabletIds, operativeSlots, virtualColumns, annOrderKeys, annLimit); + } + + /** + * add virtual column to olap scan. + * @param virtualColumns generated virtual columns + * @return scan with virtual columns + */ + public LogicalOlapScan withVirtualColumnsAndTopN( + List<NamedExpression> virtualColumns, + List<OrderKey> orderKeys, + long limit) { + LogicalProperties logicalProperties = getLogicalProperties(); + List<Slot> output = Lists.newArrayList(logicalProperties.getOutput()); + output.addAll(virtualColumns.stream().map(NamedExpression::toSlot).collect(Collectors.toList())); + logicalProperties = new LogicalProperties(() -> output, this::computeDataTrait); + return new LogicalOlapScan(relationId, (Table) table, qualifier, + groupExpression, Optional.of(logicalProperties), + selectedPartitionIds, partitionPruned, selectedTabletIds, + selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, + hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, + manuallySpecifiedTabletIds, operativeSlots, virtualColumns, orderKeys, Optional.of(limit)); } @Override @@ -523,6 +577,14 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan return virtualColumns; } + public List<OrderKey> getAnnOrderKeys() { + return annOrderKeys; + } + + public Optional<Long> getAnnLimit() { + return annLimit; + } + private List<SlotReference> createSlotsVectorized(List<Column> columns) { List<String> qualified = qualified(); SlotReference[] slots = new SlotReference[columns.size()]; @@ -682,25 +744,7 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan selectedPartitionIds, partitionPruned, selectedTabletIds, selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, - manuallySpecifiedTabletIds, operativeSlots, virtualColumns); - } - - /** - * add virtual column to olap scan. - * @param virtualColumns generated virtual columns - * @return scan with virtual columns - */ - public LogicalOlapScan withVirtualColumns(List<NamedExpression> virtualColumns) { - LogicalProperties logicalProperties = getLogicalProperties(); - List<Slot> output = Lists.newArrayList(logicalProperties.getOutput()); - output.addAll(virtualColumns.stream().map(NamedExpression::toSlot).collect(Collectors.toList())); - logicalProperties = new LogicalProperties(() -> output, this::computeDataTrait); - return new LogicalOlapScan(relationId, (Table) table, qualifier, - groupExpression, Optional.of(logicalProperties), - selectedPartitionIds, partitionPruned, selectedTabletIds, - selectedIndexId, indexSelected, preAggStatus, manuallySpecifiedPartitions, - hints, cacheSlotWithSlotName, tableSample, directMvScan, colToSubPathsMap, - manuallySpecifiedTabletIds, operativeSlots, virtualColumns); + manuallySpecifiedTabletIds, operativeSlots, virtualColumns, annOrderKeys, annLimit); } Map<Slot, Slot> constructReplaceMap(MTMV mtmv) { @@ -757,13 +801,15 @@ public class LogicalOlapScan extends LogicalCatalogRelation implements OlapScan && Objects.equals(selectedPartitionIds, that.selectedPartitionIds) && Objects.equals(hints, that.hints) && Objects.equals(tableSample, that.tableSample) - && Objects.equals(virtualColumns, that.virtualColumns); + && Objects.equals(virtualColumns, that.virtualColumns) + && Objects.equals(annOrderKeys, that.annOrderKeys) + && Objects.equals(annLimit, that.annLimit); } @Override public int hashCode() { return Objects.hash(super.hashCode(), selectedIndexId, indexSelected, preAggStatus, cacheSlotWithSlotName, selectedTabletIds, partitionPruned, manuallySpecifiedTabletIds, manuallySpecifiedPartitions, - selectedPartitionIds, hints, tableSample, virtualColumns); + selectedPartitionIds, hints, tableSample, virtualColumns, annOrderKeys, annLimit); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java index 27e242a4a70..70ed13c6ffd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOlapScan.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; +import org.apache.doris.nereids.properties.OrderKey; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -64,6 +65,10 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca // use for virtual slot private final List<NamedExpression> virtualColumns; + // use for ann push down + private final List<OrderKey> annOrderKeys; + private final Optional<Long> annLimit; + /** * Constructor for PhysicalOlapScan. */ @@ -71,12 +76,13 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca List<Long> selectedTabletIds, List<Long> selectedPartitionIds, DistributionSpec distributionSpec, PreAggStatus preAggStatus, List<Slot> baseOutputs, Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, - Optional<TableSample> tableSample, List<Slot> operativeSlots, List<NamedExpression> virtualColumns) { + Optional<TableSample> tableSample, List<Slot> operativeSlots, List<NamedExpression> virtualColumns, + List<OrderKey> annOrderKeys, Optional<Long> annLimit) { this(id, olapTable, qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, logicalProperties, null, - null, tableSample, operativeSlots, virtualColumns); + null, tableSample, operativeSlots, virtualColumns, annOrderKeys, annLimit); } /** @@ -88,7 +94,8 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca Optional<GroupExpression> groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, Optional<TableSample> tableSample, - Collection<Slot> operativeSlots, List<NamedExpression> virtualColumns) { + Collection<Slot> operativeSlots, List<NamedExpression> virtualColumns, + List<OrderKey> annOrderKeys, Optional<Long> annLimit) { super(id, PlanType.PHYSICAL_OLAP_SCAN, olapTable, qualifier, groupExpression, logicalProperties, physicalProperties, statistics, operativeSlots); this.selectedIndexId = selectedIndexId; @@ -100,6 +107,8 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca this.tableSample = tableSample; this.operativeSlots = ImmutableList.copyOf(operativeSlots); this.virtualColumns = ImmutableList.copyOf(virtualColumns); + this.annOrderKeys = ImmutableList.copyOf(annOrderKeys); + this.annLimit = annLimit; } @Override @@ -147,6 +156,14 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca return virtualColumns.stream().collect(Collectors.toMap(e -> e.toSlot().getExprId(), e -> e)); } + public List<OrderKey> getAnnOrderKeys() { + return annOrderKeys; + } + + public Optional<Long> getAnnLimit() { + return annLimit; + } + @Override public String getFingerprint() { String partitions = ""; @@ -198,18 +215,23 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca return false; } PhysicalOlapScan olapScan = (PhysicalOlapScan) o; - return selectedIndexId == olapScan.selectedIndexId && Objects.equals(distributionSpec, - olapScan.distributionSpec) && Objects.equals(selectedTabletIds, olapScan.selectedTabletIds) + return selectedIndexId == olapScan.selectedIndexId + && Objects.equals(distributionSpec, olapScan.distributionSpec) + && Objects.equals(selectedTabletIds, olapScan.selectedTabletIds) && Objects.equals(selectedPartitionIds, olapScan.selectedPartitionIds) - && Objects.equals(preAggStatus, olapScan.preAggStatus) && Objects.equals(baseOutputs, - olapScan.baseOutputs); + && Objects.equals(preAggStatus, olapScan.preAggStatus) + && Objects.equals(baseOutputs, olapScan.baseOutputs) + && Objects.equals(tableSample, olapScan.tableSample) + && Objects.equals(operativeSlots, olapScan.operativeSlots) + && Objects.equals(virtualColumns, olapScan.virtualColumns) + && Objects.equals(annOrderKeys, olapScan.annOrderKeys) + && Objects.equals(annLimit, olapScan.annLimit); } @Override public int hashCode() { return Objects.hash(super.hashCode(), distributionSpec, selectedIndexId, selectedTabletIds, - selectedPartitionIds, - preAggStatus, baseOutputs); + selectedPartitionIds, preAggStatus, baseOutputs, virtualColumns, annOrderKeys, annLimit); } @Override @@ -221,7 +243,8 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca public PhysicalOlapScan withGroupExpression(Optional<GroupExpression> groupExpression) { return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, - groupExpression, getLogicalProperties(), tableSample, operativeSlots, virtualColumns); + groupExpression, getLogicalProperties(), tableSample, operativeSlots, virtualColumns, annOrderKeys, + annLimit); } @Override @@ -229,7 +252,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca Optional<LogicalProperties> logicalProperties, List<Plan> children) { return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, - logicalProperties.get(), tableSample, operativeSlots, virtualColumns); + logicalProperties.get(), tableSample, operativeSlots, virtualColumns, annOrderKeys, annLimit); } @Override @@ -237,7 +260,8 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalOlapScan(relationId, getTable(), qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, - getLogicalProperties(), physicalProperties, statistics, tableSample, operativeSlots, virtualColumns); + getLogicalProperties(), physicalProperties, statistics, tableSample, operativeSlots, + virtualColumns, annOrderKeys, annLimit); } @Override @@ -263,7 +287,7 @@ public class PhysicalOlapScan extends PhysicalCatalogRelation implements OlapSca return new PhysicalOlapScan(relationId, (OlapTable) table, qualifier, selectedIndexId, selectedTabletIds, selectedPartitionIds, distributionSpec, preAggStatus, baseOutputs, groupExpression, getLogicalProperties(), getPhysicalProperties(), statistics, - tableSample, operativeSlots, virtualColumns); + tableSample, operativeSlots, virtualColumns, annOrderKeys, annLimit); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java index 68e11ed5b22..a897631964f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OlapScanNode.java @@ -209,6 +209,8 @@ public class OlapScanNode extends ScanNode { protected List<Expr> rewrittenProjectList; private long maxVersion = -1L; + private SortInfo annSortInfo = null; + private long annSortLimit = -1; // cached for prepared statement to quickly prune partition // only used in short circuit plan at present @@ -315,6 +317,14 @@ public class OlapScanNode extends ScanNode { this.sortLimit = sortLimit; } + public void setAnnSortInfo(SortInfo annSortInfo) { + this.annSortInfo = annSortInfo; + } + + public void setAnnSortLimit(long annSortLimit) { + this.annSortLimit = annSortLimit; + } + public Collection<Long> getSelectedPartitionIds() { return selectedPartitionIds; } @@ -1380,6 +1390,15 @@ public class OlapScanNode extends ScanNode { if (sortLimit != -1) { output.append(prefix).append("SORT LIMIT: ").append(sortLimit).append("\n"); } + if (annSortInfo != null) { + output.append(prefix).append("ANN SORT INFO:\n"); + annSortInfo.getOrderingExprs().forEach(expr -> { + output.append(prefix).append(prefix).append(expr.toSql()).append("\n"); + }); + } + if (annSortLimit != -1) { + output.append(prefix).append("ANN SORT LIMIT: ").append(annSortLimit).append("\n"); + } if (useTopnFilter()) { String topnFilterSources = String.join(",", topnFilterSortNodes.stream() @@ -1581,6 +1600,19 @@ public class OlapScanNode extends ScanNode { if (sortLimit != -1) { msg.olap_scan_node.setSortLimit(sortLimit); } + if (annSortInfo != null) { + TSortInfo tAnnSortInfo = new TSortInfo( + Expr.treesToThrift(annSortInfo.getOrderingExprs()), + annSortInfo.getIsAscOrder(), + annSortInfo.getNullsFirst()); + if (annSortInfo.getSortTupleSlotExprs() != null) { + tAnnSortInfo.setSortTupleSlotExprs(Expr.treesToThrift(annSortInfo.getSortTupleSlotExprs())); + } + msg.olap_scan_node.setAnnSortInfo(tAnnSortInfo); + } + if (annSortLimit != -1) { + msg.olap_scan_node.setAnnSortLimit(annSortLimit); + } msg.olap_scan_node.setKeyType(olapTable.getKeysType().toThrift()); String tableName = olapTable.getName(); if (selectedIndexId != -1) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index 86a0b366b8e..4143b1bb228 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -68,7 +68,7 @@ public class PhysicalPlanTranslatorTest { PhysicalOlapScan scan = new PhysicalOlapScan(StatementScopeIdGenerator.newRelationId(), t1, qualifier, t1.getBaseIndexId(), Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, Optional.empty(), - ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty()); Literal t1FilterRight = new IntegerLiteral(1); Expression t1FilterExpr = new GreaterThan(col1, t1FilterRight); PhysicalFilter<PhysicalOlapScan> filter = diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java index 8237b38209e..2401ed14e02 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/MergeProjectPostProcessTest.java @@ -78,7 +78,8 @@ public class MergeProjectPostProcessTest { LogicalProperties t1Properties = new LogicalProperties(() -> t1Output, () -> DataTrait.EMPTY_TRAIT); PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), - Optional.empty(), t1Properties, Optional.empty(), ImmutableList.of(), ImmutableList.of()); + Optional.empty(), t1Properties, Optional.empty(), ImmutableList.of(), + ImmutableList.of(), ImmutableList.of(), Optional.empty()); Alias x = new Alias(a, "x"); List<NamedExpression> projList3 = Lists.newArrayList(x, b, c); PhysicalProject proj3 = new PhysicalProject(projList3, placeHolder, scan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java index 380316e04b1..e4919e98339 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/postprocess/PushDownFilterThroughProjectTest.java @@ -93,7 +93,7 @@ public class PushDownFilterThroughProjectTest { PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, - Optional.empty(), ImmutableList.of(), ImmutableList.of()); + Optional.empty(), ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty()); Alias x = new Alias(a, "x"); List<NamedExpression> projList3 = Lists.newArrayList(x, b, c); PhysicalProject proj3 = new PhysicalProject(projList3, placeHolder, scan); @@ -132,7 +132,7 @@ public class PushDownFilterThroughProjectTest { PhysicalOlapScan scan = new PhysicalOlapScan(RelationId.createGenerator().getNextId(), t1, qualifier, 0L, Collections.emptyList(), Collections.emptyList(), null, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), t1Properties, - Optional.empty(), new ArrayList<>(), ImmutableList.of()); + Optional.empty(), new ArrayList<>(), ImmutableList.of(), ImmutableList.of(), Optional.empty()); Alias x = new Alias(a, "x"); List<NamedExpression> projList3 = Lists.newArrayList(x, b, c); PhysicalProject proj3 = new PhysicalProject(projList3, placeHolder, scan); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java index 5b8cad01c05..43e69d74d80 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanEqualsTest.java @@ -268,20 +268,20 @@ class PlanEqualsTest { 1L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties, Optional.empty(), - ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty()); PhysicalOlapScan expected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("a"), 1L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties, Optional.empty(), - ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty()); Assertions.assertEquals(expected, actual); PhysicalOlapScan unexpected = new PhysicalOlapScan(id, olapTable, Lists.newArrayList("b"), 12345L, selectedTabletId, olapTable.getPartitionIds(), distributionSpecHash, PreAggStatus.on(), ImmutableList.of(), Optional.empty(), logicalProperties, Optional.empty(), - ImmutableList.of(), ImmutableList.of()); + ImmutableList.of(), ImmutableList.of(), ImmutableList.of(), Optional.empty()); Assertions.assertNotEquals(unexpected, actual); } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org