morrySnow commented on code in PR #14378: URL: https://github.com/apache/doris/pull/14378#discussion_r1027668664
########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java: ########## @@ -124,23 +127,33 @@ public int hashCode() { @Override public Plan withGroupExpression(Optional<GroupExpression> groupExpression) { return new LogicalOlapScan(id, table, qualifier, groupExpression, Optional.of(getLogicalProperties()), - selectedPartitionIds, partitionPruned, candidateIndexIds, indexSelected, preAggStatus); + selectedPartitionIds, partitionPruned, selectedTabletId, tabletPruned, + candidateIndexIds, indexSelected, preAggStatus); } @Override public LogicalOlapScan withLogicalProperties(Optional<LogicalProperties> logicalProperties) { - return new LogicalOlapScan(id, table, qualifier, Optional.empty(), logicalProperties, selectedPartitionIds, - partitionPruned, candidateIndexIds, indexSelected, preAggStatus); + return new LogicalOlapScan(id, table, qualifier, Optional.empty(), logicalProperties, + selectedPartitionIds, partitionPruned, selectedTabletId, tabletPruned, + candidateIndexIds, indexSelected, preAggStatus); } public LogicalOlapScan withSelectedPartitionId(List<Long> selectedPartitionId) { return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), - selectedPartitionId, true, candidateIndexIds, indexSelected, preAggStatus); + selectedPartitionId, true, selectedTabletId, tabletPruned, + candidateIndexIds, indexSelected, preAggStatus); } public LogicalOlapScan withMaterializedIndexSelected(PreAggStatus preAgg, List<Long> candidateIndexIds) { return new LogicalOlapScan(id, table, qualifier, Optional.empty(), Optional.of(getLogicalProperties()), - selectedPartitionIds, partitionPruned, candidateIndexIds, true, preAgg); + selectedPartitionIds, partitionPruned, selectedTabletId, tabletPruned, + candidateIndexIds, true, preAgg); + } + + public LogicalOlapScan withSelectedTabletId(ImmutableList<Long> selectedTabletId) { Review Comment: ```suggestion public LogicalOlapScan withSelectedTabletIds(ImmutableList<Long> selectedTabletIds) { ``` ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java: ########## @@ -57,31 +58,33 @@ public LogicalOlapScan(RelationId id, OlapTable table) { public LogicalOlapScan(RelationId id, OlapTable table, List<String> qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - table.getPartitionIds(), false, ImmutableList.of(), false, PreAggStatus.on()); + table.getPartitionIds(), false, ImmutableList.of(), false, + ImmutableList.of(), false, PreAggStatus.on()); } public LogicalOlapScan(RelationId id, Table table, List<String> qualifier) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false, PreAggStatus.on()); + ((OlapTable) table).getPartitionIds(), false, ImmutableList.of(), false, + ImmutableList.of(), false, PreAggStatus.on()); } /** * Constructor for LogicalOlapScan. */ public LogicalOlapScan(RelationId id, Table table, List<String> qualifier, Optional<GroupExpression> groupExpression, Optional<LogicalProperties> logicalProperties, - List<Long> selectedPartitionIdList, boolean partitionPruned, List<Long> candidateIndexIds, - boolean indexSelected, PreAggStatus preAggStatus) { + List<Long> selectedPartitionIdList, boolean partitionPruned, + ImmutableList<Long> selectedTabletIdList, boolean tabletPruned, Review Comment: ```suggestion List<Long> selectedPartitionIds, boolean partitionPruned, ImmutableList<Long> selectedTabletIds, boolean tabletPruned, ``` ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java: ########## @@ -160,6 +173,10 @@ public long getSelectedIndexId() { return selectedIndexId; } + public ImmutableList<Long> getCandidateIndexIds() { + return candidateIndexIds; + } + Review Comment: remove it ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java: ########## @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionColumnFilterConverter; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.planner.HashDistributionPruner; +import org.apache.doris.planner.PartitionColumnFilter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * prune bucket + */ +public class PruneOlapScanTablet extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalFilter(logicalOlapScan()) + .thenApply(ctx -> { Review Comment: if u do not use ctx, then u can use `then` instead of `thenApply` ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java: ########## @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionColumnFilterConverter; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.planner.HashDistributionPruner; +import org.apache.doris.planner.PartitionColumnFilter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * prune bucket + */ +public class PruneOlapScanTablet extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalFilter(logicalOlapScan()) + .thenApply(ctx -> { + LogicalFilter<LogicalOlapScan> filter = ctx.root; + LogicalOlapScan olapScan = filter.child(); + OlapTable table = olapScan.getTable(); + List<Long> indexList = Lists.newArrayList(); + for (Long id : olapScan.getSelectedPartitionIds()) { + Partition partition = table.getPartition(id); + MaterializedIndex index = partition.getIndex(olapScan.getSelectedIndexId()); + indexList.addAll(getPrunedTablet(olapScan, filter.getConjuncts(), + index, table.getDefaultDistributionInfo())); + } + return filter.withChildren(olapScan.withSelectedTabletId(ImmutableList.copyOf(indexList))); + }).toRule(RuleType.OLAP_SCAN_TABLET_PRUNE); + } + + private Collection<Long> getPrunedTablet(LogicalOlapScan olapScan, List<Expression> exprs, + MaterializedIndex index, DistributionInfo info) { + if (info.getType() == DistributionInfoType.HASH) { Review Comment: ```suggestion if (info.getType() != DistributionInfoType.HASH) { return index.getTabletIdsInOrder(); } else { ... } ``` ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionColumnFilterConverter.java: ########## @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.visitor; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.NullLiteral; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.planner.PartitionColumnFilter; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * expression column filter converter + */ +public class ExpressionColumnFilterConverter + extends DefaultExpressionVisitor<Expression, Map<String, PartitionColumnFilter>> { + public static ExpressionColumnFilterConverter INSTANCE = new ExpressionColumnFilterConverter(); + + public static void convert(Expression expr, Map<String, PartitionColumnFilter> context) { Review Comment: rename context to columnFilters ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionColumnFilterConverter.java: ########## @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.expressions.visitor; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.LiteralExpr; +import org.apache.doris.analysis.NullLiteral; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.nereids.trees.expressions.ComparisonPredicate; +import org.apache.doris.nereids.trees.expressions.EqualTo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.GreaterThan; +import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; +import org.apache.doris.nereids.trees.expressions.InPredicate; +import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.LessThan; +import org.apache.doris.nereids.trees.expressions.LessThanEqual; +import org.apache.doris.nereids.trees.expressions.NullSafeEqual; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.planner.PartitionColumnFilter; + +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +/** + * expression column filter converter + */ +public class ExpressionColumnFilterConverter + extends DefaultExpressionVisitor<Expression, Map<String, PartitionColumnFilter>> { + public static ExpressionColumnFilterConverter INSTANCE = new ExpressionColumnFilterConverter(); + + public static void convert(Expression expr, Map<String, PartitionColumnFilter> context) { + expr.accept(INSTANCE, context); + } + + @Override + public Expression visitComparisonPredicate(ComparisonPredicate predicate, + Map<String, PartitionColumnFilter> context) { + if (predicate instanceof NullSafeEqual) { + return null; + } + PartitionColumnFilter filter = new PartitionColumnFilter(); + LiteralExpr literal = ((Literal) predicate.right()).toLegacyLiteral(); + if (predicate instanceof EqualTo) { + setFilter(filter, literal, true, literal, true); Review Comment: maybe a filter generator is better than create filter outside `setFilter` method and modified filter in it ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOlapScan.java: ########## @@ -124,23 +127,33 @@ public int hashCode() { @Override public Plan withGroupExpression(Optional<GroupExpression> groupExpression) { return new LogicalOlapScan(id, table, qualifier, groupExpression, Optional.of(getLogicalProperties()), - selectedPartitionIds, partitionPruned, candidateIndexIds, indexSelected, preAggStatus); + selectedPartitionIds, partitionPruned, selectedTabletId, tabletPruned, + candidateIndexIds, indexSelected, preAggStatus); } @Override public LogicalOlapScan withLogicalProperties(Optional<LogicalProperties> logicalProperties) { - return new LogicalOlapScan(id, table, qualifier, Optional.empty(), logicalProperties, selectedPartitionIds, - partitionPruned, candidateIndexIds, indexSelected, preAggStatus); + return new LogicalOlapScan(id, table, qualifier, Optional.empty(), logicalProperties, + selectedPartitionIds, partitionPruned, selectedTabletId, tabletPruned, + candidateIndexIds, indexSelected, preAggStatus); } public LogicalOlapScan withSelectedPartitionId(List<Long> selectedPartitionId) { Review Comment: ```suggestion public LogicalOlapScan withSelectedPartitionIds(List<Long> selectedPartitionIds) { ``` ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java: ########## @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionColumnFilterConverter; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.planner.HashDistributionPruner; +import org.apache.doris.planner.PartitionColumnFilter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * prune bucket + */ +public class PruneOlapScanTablet extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalFilter(logicalOlapScan()) + .thenApply(ctx -> { + LogicalFilter<LogicalOlapScan> filter = ctx.root; + LogicalOlapScan olapScan = filter.child(); + OlapTable table = olapScan.getTable(); + List<Long> indexList = Lists.newArrayList(); Review Comment: ```suggestion List<Long> selectedTabletIds = Lists.newArrayList(); ``` ########## fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java: ########## @@ -313,5 +317,30 @@ public static List<Expression> extractCoveredConjunction(List<Expression> predic } return coveredPredicates; } + + /** + * check and maybe commute for predications except not pred. + */ + public static Expression checkAndMaybeCommute(Expression expression) { Review Comment: return option.empty instead of null ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/logical/PruneOlapScanTablet.java: ########## @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite.logical; + +import org.apache.doris.catalog.DistributionInfo; +import org.apache.doris.catalog.DistributionInfo.DistributionInfoType; +import org.apache.doris.catalog.HashDistributionInfo; +import org.apache.doris.catalog.MaterializedIndex; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.Partition; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.rules.rewrite.OneRewriteRuleFactory; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.visitor.ExpressionColumnFilterConverter; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.util.ExpressionUtils; +import org.apache.doris.planner.HashDistributionPruner; +import org.apache.doris.planner.PartitionColumnFilter; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; + +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * prune bucket + */ +public class PruneOlapScanTablet extends OneRewriteRuleFactory { + + @Override + public Rule build() { + return logicalFilter(logicalOlapScan()) + .thenApply(ctx -> { + LogicalFilter<LogicalOlapScan> filter = ctx.root; + LogicalOlapScan olapScan = filter.child(); + OlapTable table = olapScan.getTable(); + List<Long> indexList = Lists.newArrayList(); + for (Long id : olapScan.getSelectedPartitionIds()) { + Partition partition = table.getPartition(id); + MaterializedIndex index = partition.getIndex(olapScan.getSelectedIndexId()); + indexList.addAll(getPrunedTablet(olapScan, filter.getConjuncts(), + index, table.getDefaultDistributionInfo())); + } + return filter.withChildren(olapScan.withSelectedTabletId(ImmutableList.copyOf(indexList))); + }).toRule(RuleType.OLAP_SCAN_TABLET_PRUNE); + } + + private Collection<Long> getPrunedTablet(LogicalOlapScan olapScan, List<Expression> exprs, Review Comment: ```suggestion private Collection<Long> getSelectedTabletIds(LogicalOlapScan olapScan, List<Expression> exprs, ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org