morrySnow commented on code in PR #33647: URL: https://github.com/apache/doris/pull/33647#discussion_r1568188532
########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java: ########## @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.pattern.MatchingContext; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * rewrite func(para) over (partition by unique_keys) + * 1. func() is count(non-null) or rank/dense_rank/row_number -> 1 + * 2. func(para) is min/max/sum/avg/first_value/last_value -> para + * e.g + * select max(c1) over(partition by pk) from t1; + * -> select c1 from t1; + * */ +public class SimplifyWindowExpression extends OneRewriteRuleFactory { Review Comment: add annotation `@DependsRules` for denpends on normalize window ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java: ########## @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.pattern.MatchingContext; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * rewrite func(para) over (partition by unique_keys) + * 1. func() is count(non-null) or rank/dense_rank/row_number -> 1 + * 2. func(para) is min/max/sum/avg/first_value/last_value -> para + * e.g + * select max(c1) over(partition by pk) from t1; + * -> select c1 from t1; + * */ +public class SimplifyWindowExpression extends OneRewriteRuleFactory { + private static final String COUNT = "count"; + private static final ImmutableSet<String> REWRRITE_TO_CONST_WINDOW_FUNCTIONS = + ImmutableSet.of("rank", "dense_rank", "row_number"); + private static final ImmutableSet<String> REWRRITE_TO_SLOT_WINDOW_FUNCTIONS = + ImmutableSet.of("min", "max", "sum", "avg", "first_value", "last_value"); + + @Override + public Rule build() { + return logicalWindow(any()).thenApply(this::simplify) + .toRule(RuleType.SIMPLIFY_WINDOW_EXPRESSION); + } + + private Plan simplify(MatchingContext<LogicalWindow<Plan>> ctx) { + LogicalWindow<Plan> window = ctx.root; + ImmutableList.Builder<NamedExpression> projectionsBuilder = ImmutableList.builder(); + ImmutableList.Builder<NamedExpression> remainWindowExpression = ImmutableList.builder(); + List<NamedExpression> windowExpressions = window.getWindowExpressions(); + for (NamedExpression expr : windowExpressions) { + Alias alias = (Alias) expr; + WindowExpression windowExpression = (WindowExpression) alias.child(); + if (!windowExpression.getOrderKeys().isEmpty()) { + remainWindowExpression.add(expr); Review Comment: if partition key is unique, order key is not use anymore, so why remain them? ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java: ########## @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.pattern.MatchingContext; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * rewrite func(para) over (partition by unique_keys) + * 1. func() is count(non-null) or rank/dense_rank/row_number -> 1 + * 2. func(para) is min/max/sum/avg/first_value/last_value -> para + * e.g + * select max(c1) over(partition by pk) from t1; + * -> select c1 from t1; + * */ +public class SimplifyWindowExpression extends OneRewriteRuleFactory { + private static final String COUNT = "count"; + private static final ImmutableSet<String> REWRRITE_TO_CONST_WINDOW_FUNCTIONS = + ImmutableSet.of("rank", "dense_rank", "row_number"); + private static final ImmutableSet<String> REWRRITE_TO_SLOT_WINDOW_FUNCTIONS = + ImmutableSet.of("min", "max", "sum", "avg", "first_value", "last_value"); + + @Override + public Rule build() { + return logicalWindow(any()).thenApply(this::simplify) + .toRule(RuleType.SIMPLIFY_WINDOW_EXPRESSION); + } + + private Plan simplify(MatchingContext<LogicalWindow<Plan>> ctx) { + LogicalWindow<Plan> window = ctx.root; + ImmutableList.Builder<NamedExpression> projectionsBuilder = ImmutableList.builder(); + ImmutableList.Builder<NamedExpression> remainWindowExpression = ImmutableList.builder(); + List<NamedExpression> windowExpressions = window.getWindowExpressions(); + for (NamedExpression expr : windowExpressions) { + Alias alias = (Alias) expr; + WindowExpression windowExpression = (WindowExpression) alias.child(); + if (!windowExpression.getOrderKeys().isEmpty()) { + remainWindowExpression.add(expr); + continue; + } + if (windowExpression.getPartitionKeys().stream().anyMatch(( + partitionKey -> partitionKey.getDataType().isOnlyMetricType()))) { + continue; + } + // after normalize window, partition key must be slot + List<Slot> partitionSlots = (List<Slot>) (List) windowExpression.getPartitionKeys(); + Set<Slot> partitionSlotSet = new HashSet<>(partitionSlots); + if (!window.getLogicalProperties().getFunctionalDependencies().isUnique(partitionSlotSet)) { + remainWindowExpression.add(expr); + continue; + } + Expression function = windowExpression.getFunction(); + if (function instanceof BoundFunction) { + BoundFunction boundFunction = (BoundFunction) function; + String name = ((BoundFunction) function).getName(); + if ((name.equals(COUNT) && boundFunction.child(0).notNullable()) + || REWRRITE_TO_CONST_WINDOW_FUNCTIONS.contains(name)) { + projectionsBuilder.add(new Alias(alias.getExprId(), new TinyIntLiteral((byte) 1), alias.getName())); + } else if (REWRRITE_TO_SLOT_WINDOW_FUNCTIONS.contains(name)) { + projectionsBuilder.add(new Alias(alias.getExprId(), boundFunction.child(0), alias.getName())); + } else { + remainWindowExpression.add(expr); + } + } else { + remainWindowExpression.add(expr); + } + } + List<NamedExpression> projections = projectionsBuilder.build(); + List<NamedExpression> remainWindow = remainWindowExpression.build(); + + if (projections.isEmpty()) { + return window; + } else if (remainWindow.isEmpty()) { + return new LogicalProject(projections, window.child(0)); + } else { + return new LogicalProject(projections, window.withExpression(remainWindow, + window.child(0))); + } + } + +} + + + + + + + + + + + + + + + Review Comment: what's this? ########## fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/SimplifyWindowExpression.java: ########## @@ -0,0 +1,129 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.nereids.pattern.MatchingContext; +import org.apache.doris.nereids.rules.Rule; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.WindowExpression; +import org.apache.doris.nereids.trees.expressions.functions.BoundFunction; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalWindow; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; + +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +/** + * rewrite func(para) over (partition by unique_keys) + * 1. func() is count(non-null) or rank/dense_rank/row_number -> 1 + * 2. func(para) is min/max/sum/avg/first_value/last_value -> para + * e.g + * select max(c1) over(partition by pk) from t1; + * -> select c1 from t1; + * */ +public class SimplifyWindowExpression extends OneRewriteRuleFactory { + private static final String COUNT = "count"; + private static final ImmutableSet<String> REWRRITE_TO_CONST_WINDOW_FUNCTIONS = + ImmutableSet.of("rank", "dense_rank", "row_number"); + private static final ImmutableSet<String> REWRRITE_TO_SLOT_WINDOW_FUNCTIONS = + ImmutableSet.of("min", "max", "sum", "avg", "first_value", "last_value"); + + @Override + public Rule build() { + return logicalWindow(any()).thenApply(this::simplify) + .toRule(RuleType.SIMPLIFY_WINDOW_EXPRESSION); + } + + private Plan simplify(MatchingContext<LogicalWindow<Plan>> ctx) { + LogicalWindow<Plan> window = ctx.root; + ImmutableList.Builder<NamedExpression> projectionsBuilder = ImmutableList.builder(); + ImmutableList.Builder<NamedExpression> remainWindowExpression = ImmutableList.builder(); + List<NamedExpression> windowExpressions = window.getWindowExpressions(); + for (NamedExpression expr : windowExpressions) { + Alias alias = (Alias) expr; + WindowExpression windowExpression = (WindowExpression) alias.child(); + if (!windowExpression.getOrderKeys().isEmpty()) { + remainWindowExpression.add(expr); + continue; + } + if (windowExpression.getPartitionKeys().stream().anyMatch(( + partitionKey -> partitionKey.getDataType().isOnlyMetricType()))) { + continue; + } + // after normalize window, partition key must be slot + List<Slot> partitionSlots = (List<Slot>) (List) windowExpression.getPartitionKeys(); + Set<Slot> partitionSlotSet = new HashSet<>(partitionSlots); + if (!window.getLogicalProperties().getFunctionalDependencies().isUnique(partitionSlotSet)) { + remainWindowExpression.add(expr); + continue; + } + Expression function = windowExpression.getFunction(); + if (function instanceof BoundFunction) { + BoundFunction boundFunction = (BoundFunction) function; + String name = ((BoundFunction) function).getName(); + if ((name.equals(COUNT) && boundFunction.child(0).notNullable()) + || REWRRITE_TO_CONST_WINDOW_FUNCTIONS.contains(name)) { + projectionsBuilder.add(new Alias(alias.getExprId(), new TinyIntLiteral((byte) 1), alias.getName())); + } else if (REWRRITE_TO_SLOT_WINDOW_FUNCTIONS.contains(name)) { + projectionsBuilder.add(new Alias(alias.getExprId(), boundFunction.child(0), alias.getName())); + } else { + remainWindowExpression.add(expr); + } + } else { + remainWindowExpression.add(expr); + } + } + List<NamedExpression> projections = projectionsBuilder.build(); + List<NamedExpression> remainWindow = remainWindowExpression.build(); + + if (projections.isEmpty()) { + return window; + } else if (remainWindow.isEmpty()) { + return new LogicalProject(projections, window.child(0)); + } else { + return new LogicalProject(projections, window.withExpression(remainWindow, + window.child(0))); Review Comment: do u have case to test this branch? i think this is not right because project should contain remain window node's output -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
