morrySnow commented on code in PR #31812: URL: https://github.com/apache/doris/pull/31812#discussion_r1560992485
########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java: ########## @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/Expr.java +// and modified by Doris + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionParams; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.mtmv.MTMVPartitionExprFactory; +import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * MTMVPartitionDefinition + */ +public class MTMVPartitionDefinition { + private MTMVPartitionType partitionType; + private String partitionCol; + private Expression functionCallExpression; + + /** + * analyzeAndTransferToMTMVPartitionInfo + * + * @param planner planner + * @param ctx ctx + * @param logicalQuery logicalQuery + * @return MTMVPartitionInfo + */ + public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx, + LogicalPlan logicalQuery) { + MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType); + if (this.partitionType == MTMVPartitionType.SELF_MANAGE) { + return mtmvPartitionInfo; + } + String partitionColName; + if (this.partitionType == MTMVPartitionType.EXPR) { + Expr expr = convertToLegacyAutoPartitionExprs(Lists.newArrayList(functionCallExpression)) + .get(0); Review Comment: not wrap here ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRelatedPartitionDescRollUpGenerator.java: ########## @@ -0,0 +1,140 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; + +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; + +/** + * Roll up some partitions into one partition + */ +public class MTMVRelatedPartitionDescRollUpGenerator implements MTMVRelatedPartitionDescGeneratorService { + + @Override + public void apply(MTMVPartitionInfo mvPartitionInfo, Map<String, String> mvProperties, + RelatedPartitionDescResult lastResult) throws AnalysisException { + if (mvPartitionInfo.getPartitionType() != MTMVPartitionType.EXPR) { + return; + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + lastResult.setDescs(rollUpRange(lastResult.getDescs(), mvPartitionInfo)); + } else if (partitionType == PartitionType.LIST) { + lastResult.setDescs(rollUpList(lastResult.getDescs(), mvPartitionInfo, mvProperties)); + } else { + throw new AnalysisException("only RANGE/LIST partition support roll up"); + } + } + + /** + * when related table has 3 partitions:(20200101),(20200102),(20200201) + * <p> + * if expr is `date_trunc(month)` + * then,MTMV will have 2 partitions (20200101,20200102),(20200201) + * <p> + * if expr is `date_trunc(year)` + * then,MTMV will have 1 partitions (20200101,20200102,20200201) + * + * @param relatedPartitionDescs + * @param mvPartitionInfo + * @return + * @throws AnalysisException + */ + public Map<PartitionKeyDesc, Set<Long>> rollUpList(Map<PartitionKeyDesc, Set<Long>> relatedPartitionDescs, + MTMVPartitionInfo mvPartitionInfo, Map<String, String> mvProperties) throws AnalysisException { + Map<String, Set<String>> identityToValues = Maps.newHashMap(); + Map<String, Set<Long>> identityToPartitionIds = Maps.newHashMap(); + MTMVPartitionExprService exprSerice = MTMVPartitionExprFactory.getExprSerice(mvPartitionInfo.getExpr()); + + for (Entry<PartitionKeyDesc, Set<Long>> entry : relatedPartitionDescs.entrySet()) { + String rollUpIdentity = exprSerice.getRollUpIdentity(entry.getKey(), mvProperties); Review Comment: rollUpIdentity could be null, do we process null correctly? ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java: ########## @@ -56,6 +57,15 @@ public class MTMVPartitionUtil { private static final Pattern PARTITION_NAME_PATTERN = Pattern.compile("[^a-zA-Z0-9,]"); private static final String PARTITION_NAME_PREFIX = "p_"; + private static final List<MTMVRelatedPartitionDescGeneratorService> partitionDescGenerators = ImmutableList + .of( + // It is necessary to maintain this order Review Comment: please add more comment to explain why need this order ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java: ########## @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/Expr.java +// and modified by Doris + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionParams; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.mtmv.MTMVPartitionExprFactory; +import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * MTMVPartitionDefinition + */ +public class MTMVPartitionDefinition { + private MTMVPartitionType partitionType; + private String partitionCol; + private Expression functionCallExpression; + + /** + * analyzeAndTransferToMTMVPartitionInfo + * + * @param planner planner + * @param ctx ctx + * @param logicalQuery logicalQuery + * @return MTMVPartitionInfo + */ + public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx, + LogicalPlan logicalQuery) { + MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType); + if (this.partitionType == MTMVPartitionType.SELF_MANAGE) { + return mtmvPartitionInfo; + } + String partitionColName; + if (this.partitionType == MTMVPartitionType.EXPR) { + Expr expr = convertToLegacyAutoPartitionExprs(Lists.newArrayList(functionCallExpression)) + .get(0); + partitionColName = getColNameFromExpr(expr); + mtmvPartitionInfo.setExpr(expr); + } else { + partitionColName = this.partitionCol; + } + mtmvPartitionInfo.setPartitionCol(partitionColName); + RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName); + mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn()); + mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo()); + if (this.partitionType == MTMVPartitionType.EXPR) { + try { + MTMVPartitionExprFactory.getExprSerice(mtmvPartitionInfo.getExpr()).analyze(mtmvPartitionInfo); + } catch (org.apache.doris.common.AnalysisException e) { + throw new AnalysisException(e.getMessage(), e); + } + } + return mtmvPartitionInfo; + } + + /** + * getColNameFromExpr + * + * @param expr expr + * @return String + */ + public static String getColNameFromExpr(Expr expr) { + if (!(expr instanceof FunctionCallExpr)) { + throw new AnalysisException( + "auto create partition only support function call expr is: " + + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS); + } + FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr; + List<Expr> paramsExpr = functionCallExpr.getParams().exprs(); + String name = functionCallExpr.getFnName().getFunction(); + if (MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS.contains(name)) { + for (Expr param : paramsExpr) { + if (param instanceof SlotRef) { + return ((SlotRef) param).getColumnName(); + } + } + throw new AnalysisException("can not find colName"); + } else { + throw new AnalysisException( + "auto create partition only support function call expr is: " + + MTMVPartitionInfo.MTMV_PARTITION_FUNCTIONS); + } + } + + private RelatedTableInfo getRelatedTableInfo(NereidsPlanner planner, ConnectContext ctx, LogicalPlan + logicalQuery, + String partitionColName) { + CascadesContext cascadesContext = planner.getCascadesContext(); + SessionVariable sessionVariable = cascadesContext.getConnectContext().getSessionVariable(); + Set<String> tempDisableRules = sessionVariable.getDisableNereidsRuleNames(); + // Should not make table without data to empty relation when analyze the related table, + // so add disable rules + sessionVariable.setDisableNereidsRules(CreateMTMVInfo.MTMV_PLANER_DISABLE_RULES); + cascadesContext.getStatementContext().invalidCache(SessionVariable.DISABLE_NEREIDS_RULES); Review Comment: @seawinde more than one place use disable rule. This approach has poor maintainability, we need to think of a better solution. ########## fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4: ########## @@ -225,6 +225,11 @@ refreshMethod : COMPLETE | AUTO ; +mvPartition + : LEFT_PAREN partitionKey = identifier RIGHT_PAREN + | partitionExpr = functionCallExpression + ; Review Comment: should be same as create table, function should put into paren ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java: ########## @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/Expr.java +// and modified by Doris + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionParams; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.mtmv.MTMVPartitionExprFactory; +import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * MTMVPartitionDefinition + */ +public class MTMVPartitionDefinition { + private MTMVPartitionType partitionType; + private String partitionCol; + private Expression functionCallExpression; + + /** + * analyzeAndTransferToMTMVPartitionInfo + * + * @param planner planner + * @param ctx ctx + * @param logicalQuery logicalQuery + * @return MTMVPartitionInfo + */ + public MTMVPartitionInfo analyzeAndTransferToMTMVPartitionInfo(NereidsPlanner planner, ConnectContext ctx, + LogicalPlan logicalQuery) { + MTMVPartitionInfo mtmvPartitionInfo = new MTMVPartitionInfo(partitionType); + if (this.partitionType == MTMVPartitionType.SELF_MANAGE) { + return mtmvPartitionInfo; + } + String partitionColName; + if (this.partitionType == MTMVPartitionType.EXPR) { + Expr expr = convertToLegacyAutoPartitionExprs(Lists.newArrayList(functionCallExpression)) + .get(0); + partitionColName = getColNameFromExpr(expr); + mtmvPartitionInfo.setExpr(expr); + } else { + partitionColName = this.partitionCol; + } + mtmvPartitionInfo.setPartitionCol(partitionColName); + RelatedTableInfo relatedTableInfo = getRelatedTableInfo(planner, ctx, logicalQuery, partitionColName); + mtmvPartitionInfo.setRelatedCol(relatedTableInfo.getColumn()); + mtmvPartitionInfo.setRelatedTable(relatedTableInfo.getTableInfo()); + if (this.partitionType == MTMVPartitionType.EXPR) { + try { + MTMVPartitionExprFactory.getExprSerice(mtmvPartitionInfo.getExpr()).analyze(mtmvPartitionInfo); Review Comment: Serice? what's this? ########## fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/MTMVPartitionDefinition.java: ########## @@ -0,0 +1,225 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +// This file is copied from +// https://github.com/apache/impala/blob/branch-2.9.0/fe/src/main/java/org/apache/impala/Expr.java +// and modified by Doris + +package org.apache.doris.nereids.trees.plans.commands.info; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.FunctionParams; +import org.apache.doris.analysis.SlotRef; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.mtmv.MTMVPartitionExprFactory; +import org.apache.doris.mtmv.MTMVPartitionInfo; +import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVRelatedTableIf; +import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundSlot; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils; +import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewUtils.RelatedTableInfo; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.literal.Literal; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; + +import com.google.common.collect.Lists; +import com.google.common.collect.Sets; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * MTMVPartitionDefinition + */ +public class MTMVPartitionDefinition { + private MTMVPartitionType partitionType; + private String partitionCol; + private Expression functionCallExpression; Review Comment: let them be final and let this class immutable ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java: ########## @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.PartitionExprUtil; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +public class MTMVPartitionExprDateTrunc implements MTMVPartitionExprService { + private static Set<String> timeUnits = ImmutableSet.of("year", "month", "day"); + private String timeUnit; + + public MTMVPartitionExprDateTrunc(FunctionCallExpr functionCallExpr) throws AnalysisException { + List<Expr> paramsExprs = functionCallExpr.getParams().exprs(); + if (paramsExprs.size() != 2) { + throw new AnalysisException("date_trunc params exprs size should be 2."); + } + Expr param = paramsExprs.get(1); + if (!(param instanceof StringLiteral)) { + throw new AnalysisException("date_trunc param of time unit is not string literal."); + } + this.timeUnit = param.getStringValue().toLowerCase(); + } + + @Override + public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + if (!timeUnits.contains(this.timeUnit)) { + throw new AnalysisException( + String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits)); + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + if (!partitionColumnType.isDateType()) { + throw new AnalysisException( + "partitionColumnType should be date/datetime " + + "when PartitionType is range and expr is date_trunc"); + } + } + } + + @Override + public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map<String, String> mvProperties) + throws AnalysisException { + String res = null; + Optional<String> dateFormat = getDateFormat(mvProperties); + List<List<PartitionValue>> inValues = partitionKeyDesc.getInValues(); + for (int i = 0; i < inValues.size(); i++) { + // mtmv only support one partition column + PartitionValue partitionValue = inValues.get(i).get(0); + if (partitionValue.isNullPartition()) { + throw new AnalysisException("date trunc not support null partition value"); + } + String identity = dateTrunc(partitionValue.getStringValue(), dateFormat, false).toString(); + if (i == 0) { + res = identity; + } else { + if (!Objects.equals(res, identity)) { + throw new AnalysisException( + String.format("partition values not equal, res: %s, identity: %s", res, + identity)); + } + } + } + return res; + } + + private Optional<String> getDateFormat(Map<String, String> mvProperties) { + Optional<String> dateFormat = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) + ? Optional.empty() + : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); + return dateFormat; + } + + @Override + public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc, + MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + // mtmv only support one partition column Review Comment: do we need add check here to avoid errors if we support more than one partition column and forget to do change here in future ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java: ########## @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.PartitionExprUtil; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +public class MTMVPartitionExprDateTrunc implements MTMVPartitionExprService { + private static Set<String> timeUnits = ImmutableSet.of("year", "month", "day"); + private String timeUnit; + + public MTMVPartitionExprDateTrunc(FunctionCallExpr functionCallExpr) throws AnalysisException { + List<Expr> paramsExprs = functionCallExpr.getParams().exprs(); + if (paramsExprs.size() != 2) { + throw new AnalysisException("date_trunc params exprs size should be 2."); + } + Expr param = paramsExprs.get(1); + if (!(param instanceof StringLiteral)) { + throw new AnalysisException("date_trunc param of time unit is not string literal."); + } + this.timeUnit = param.getStringValue().toLowerCase(); + } + + @Override + public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + if (!timeUnits.contains(this.timeUnit)) { + throw new AnalysisException( + String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits)); + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + if (!partitionColumnType.isDateType()) { + throw new AnalysisException( + "partitionColumnType should be date/datetime " + + "when PartitionType is range and expr is date_trunc"); + } + } + } + + @Override + public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map<String, String> mvProperties) + throws AnalysisException { + String res = null; + Optional<String> dateFormat = getDateFormat(mvProperties); + List<List<PartitionValue>> inValues = partitionKeyDesc.getInValues(); + for (int i = 0; i < inValues.size(); i++) { + // mtmv only support one partition column + PartitionValue partitionValue = inValues.get(i).get(0); + if (partitionValue.isNullPartition()) { + throw new AnalysisException("date trunc not support null partition value"); + } + String identity = dateTrunc(partitionValue.getStringValue(), dateFormat, false).toString(); + if (i == 0) { + res = identity; + } else { + if (!Objects.equals(res, identity)) { + throw new AnalysisException( + String.format("partition values not equal, res: %s, identity: %s", res, + identity)); + } + } + } + return res; + } + + private Optional<String> getDateFormat(Map<String, String> mvProperties) { + Optional<String> dateFormat = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) + ? Optional.empty() + : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); + return dateFormat; + } + + @Override + public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc, + MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + // mtmv only support one partition column + DateTimeV2Literal beginTime = dateTrunc( + partitionKeyDesc.getLowerValues().get(0).getStringValue(), + Optional.empty(), false); + + PartitionValue lowerValue = new PartitionValue(timeToStr(beginTime, partitionColumnType)); + return PartitionKeyDesc.createFixed( + Collections.singletonList(lowerValue), + Collections.singletonList( + getUpperValue(partitionKeyDesc.getUpperValues().get(0), beginTime, partitionColumnType))); Review Comment: for more clear code ```suggestion PartitionValue lowerValue = new PartitionValue(timeToStr(beginTime, partitionColumnType)); PartitionValue upperValue = getUpperValue(partitionKeyDesc.getUpperValues().get(0), beginTime, partitionColumnType)) return PartitionKeyDesc.createFixed( Collections.singletonList(lowerValue), Collections.singletonList(upperValue)); ``` ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java: ########## @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.PartitionExprUtil; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +public class MTMVPartitionExprDateTrunc implements MTMVPartitionExprService { + private static Set<String> timeUnits = ImmutableSet.of("year", "month", "day"); + private String timeUnit; + + public MTMVPartitionExprDateTrunc(FunctionCallExpr functionCallExpr) throws AnalysisException { + List<Expr> paramsExprs = functionCallExpr.getParams().exprs(); + if (paramsExprs.size() != 2) { + throw new AnalysisException("date_trunc params exprs size should be 2."); + } + Expr param = paramsExprs.get(1); + if (!(param instanceof StringLiteral)) { + throw new AnalysisException("date_trunc param of time unit is not string literal."); + } + this.timeUnit = param.getStringValue().toLowerCase(); + } + + @Override + public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + if (!timeUnits.contains(this.timeUnit)) { + throw new AnalysisException( + String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits)); + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + if (!partitionColumnType.isDateType()) { + throw new AnalysisException( + "partitionColumnType should be date/datetime " + + "when PartitionType is range and expr is date_trunc"); + } + } + } + + @Override + public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map<String, String> mvProperties) + throws AnalysisException { + String res = null; + Optional<String> dateFormat = getDateFormat(mvProperties); + List<List<PartitionValue>> inValues = partitionKeyDesc.getInValues(); + for (int i = 0; i < inValues.size(); i++) { + // mtmv only support one partition column + PartitionValue partitionValue = inValues.get(i).get(0); + if (partitionValue.isNullPartition()) { + throw new AnalysisException("date trunc not support null partition value"); + } + String identity = dateTrunc(partitionValue.getStringValue(), dateFormat, false).toString(); + if (i == 0) { + res = identity; + } else { + if (!Objects.equals(res, identity)) { + throw new AnalysisException( + String.format("partition values not equal, res: %s, identity: %s", res, + identity)); + } + } + } + return res; + } + + private Optional<String> getDateFormat(Map<String, String> mvProperties) { + Optional<String> dateFormat = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) + ? Optional.empty() + : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); + return dateFormat; + } + + @Override + public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc, + MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + // mtmv only support one partition column + DateTimeV2Literal beginTime = dateTrunc( + partitionKeyDesc.getLowerValues().get(0).getStringValue(), + Optional.empty(), false); + + PartitionValue lowerValue = new PartitionValue(timeToStr(beginTime, partitionColumnType)); + return PartitionKeyDesc.createFixed( + Collections.singletonList(lowerValue), + Collections.singletonList( + getUpperValue(partitionKeyDesc.getUpperValues().get(0), beginTime, partitionColumnType))); + } + + private PartitionValue getUpperValue(PartitionValue upperValue, DateTimeV2Literal beginTruncTime, + Type partitionColumnType) throws AnalysisException { + if (upperValue.isMax()) { + throw new AnalysisException("date trunc not support MAXVALUE partition"); + } + // begin time and end time dateTrunc should has same result + DateTimeV2Literal endTruncTime = dateTrunc(upperValue.getStringValue(), Optional.empty(), true); + if (!Objects.equals(beginTruncTime, endTruncTime)) { + throw new AnalysisException( + String.format("partition values not equal, beginTruncTime: %s, endTruncTime: %s", beginTruncTime, + endTruncTime)); + } + DateTimeV2Literal endTime = dateAdd(beginTruncTime); + return new PartitionValue(timeToStr(endTime, partitionColumnType)); + } + + private DateTimeV2Literal dateTrunc(String value, + Optional<String> dateFormat, boolean isUpper) throws AnalysisException { + DateTimeV2Literal dateTimeLiteral = strToDate(value, dateFormat); + // for (2020-01-31,2020-02-01),if not -1, lower value and upper value will not same after rollup + if (isUpper) { + dateTimeLiteral = (DateTimeV2Literal) DateTimeArithmetic.secondsSub(dateTimeLiteral, new IntegerLiteral(1)); + } + Expression expression = DateTimeExtractAndTransform.dateTrunc(dateTimeLiteral, new VarcharLiteral(timeUnit)); + if (!(expression instanceof DateTimeV2Literal)) { + throw new AnalysisException("dateTrunc() should return DateLiteral, expression: " + expression); + } + return (DateTimeV2Literal) expression; + } + + private DateTimeV2Literal strToDate(String value, + Optional<String> dateFormat) throws AnalysisException { + try { + return new DateTimeV2Literal(value); + } catch (Exception e) { + if (!dateFormat.isPresent()) { + throw e; + } + Expression strToDate = DateTimeExtractAndTransform + .strToDate(new VarcharLiteral(value), + new VarcharLiteral(dateFormat.get())); + if (strToDate instanceof DateV2Literal) { + DateV2Literal dateV2Literal = (DateV2Literal) strToDate; + return new DateTimeV2Literal(dateV2Literal.getYear(), dateV2Literal.getMonth(), dateV2Literal.getDay(), + 0, 0, 0); + } else if (strToDate instanceof DateTimeV2Literal) { + return (DateTimeV2Literal) strToDate; + } else { + throw new AnalysisException( + String.format("strToDate failed, stringValue: %s, dateFormat: %s", value, + dateFormat)); + } + } + } + + private DateTimeV2Literal dateAdd(DateTimeV2Literal value) throws AnalysisException { Review Comment: ```suggestion private DateTimeV2Literal dateIncrement(DateTimeV2Literal value) throws AnalysisException { ``` ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java: ########## @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.PartitionExprUtil; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +public class MTMVPartitionExprDateTrunc implements MTMVPartitionExprService { + private static Set<String> timeUnits = ImmutableSet.of("year", "month", "day"); + private String timeUnit; + + public MTMVPartitionExprDateTrunc(FunctionCallExpr functionCallExpr) throws AnalysisException { + List<Expr> paramsExprs = functionCallExpr.getParams().exprs(); + if (paramsExprs.size() != 2) { + throw new AnalysisException("date_trunc params exprs size should be 2."); + } + Expr param = paramsExprs.get(1); + if (!(param instanceof StringLiteral)) { + throw new AnalysisException("date_trunc param of time unit is not string literal."); + } + this.timeUnit = param.getStringValue().toLowerCase(); + } + + @Override + public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + if (!timeUnits.contains(this.timeUnit)) { + throw new AnalysisException( + String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits)); + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + if (!partitionColumnType.isDateType()) { + throw new AnalysisException( + "partitionColumnType should be date/datetime " + + "when PartitionType is range and expr is date_trunc"); + } + } + } + + @Override + public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map<String, String> mvProperties) + throws AnalysisException { + String res = null; + Optional<String> dateFormat = getDateFormat(mvProperties); + List<List<PartitionValue>> inValues = partitionKeyDesc.getInValues(); + for (int i = 0; i < inValues.size(); i++) { + // mtmv only support one partition column + PartitionValue partitionValue = inValues.get(i).get(0); + if (partitionValue.isNullPartition()) { + throw new AnalysisException("date trunc not support null partition value"); + } + String identity = dateTrunc(partitionValue.getStringValue(), dateFormat, false).toString(); + if (i == 0) { + res = identity; + } else { + if (!Objects.equals(res, identity)) { + throw new AnalysisException( + String.format("partition values not equal, res: %s, identity: %s", res, + identity)); + } + } + } + return res; + } + + private Optional<String> getDateFormat(Map<String, String> mvProperties) { + Optional<String> dateFormat = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) + ? Optional.empty() + : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); + return dateFormat; + } + + @Override + public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc, + MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + // mtmv only support one partition column + DateTimeV2Literal beginTime = dateTrunc( + partitionKeyDesc.getLowerValues().get(0).getStringValue(), + Optional.empty(), false); + + PartitionValue lowerValue = new PartitionValue(timeToStr(beginTime, partitionColumnType)); + return PartitionKeyDesc.createFixed( + Collections.singletonList(lowerValue), + Collections.singletonList( + getUpperValue(partitionKeyDesc.getUpperValues().get(0), beginTime, partitionColumnType))); + } + + private PartitionValue getUpperValue(PartitionValue upperValue, DateTimeV2Literal beginTruncTime, + Type partitionColumnType) throws AnalysisException { + if (upperValue.isMax()) { + throw new AnalysisException("date trunc not support MAXVALUE partition"); + } + // begin time and end time dateTrunc should has same result + DateTimeV2Literal endTruncTime = dateTrunc(upperValue.getStringValue(), Optional.empty(), true); + if (!Objects.equals(beginTruncTime, endTruncTime)) { + throw new AnalysisException( + String.format("partition values not equal, beginTruncTime: %s, endTruncTime: %s", beginTruncTime, + endTruncTime)); + } + DateTimeV2Literal endTime = dateAdd(beginTruncTime); + return new PartitionValue(timeToStr(endTime, partitionColumnType)); + } + + private DateTimeV2Literal dateTrunc(String value, + Optional<String> dateFormat, boolean isUpper) throws AnalysisException { + DateTimeV2Literal dateTimeLiteral = strToDate(value, dateFormat); + // for (2020-01-31,2020-02-01),if not -1, lower value and upper value will not same after rollup + if (isUpper) { + dateTimeLiteral = (DateTimeV2Literal) DateTimeArithmetic.secondsSub(dateTimeLiteral, new IntegerLiteral(1)); + } + Expression expression = DateTimeExtractAndTransform.dateTrunc(dateTimeLiteral, new VarcharLiteral(timeUnit)); + if (!(expression instanceof DateTimeV2Literal)) { + throw new AnalysisException("dateTrunc() should return DateLiteral, expression: " + expression); + } + return (DateTimeV2Literal) expression; + } + + private DateTimeV2Literal strToDate(String value, + Optional<String> dateFormat) throws AnalysisException { + try { + return new DateTimeV2Literal(value); + } catch (Exception e) { + if (!dateFormat.isPresent()) { + throw e; + } + Expression strToDate = DateTimeExtractAndTransform + .strToDate(new VarcharLiteral(value), + new VarcharLiteral(dateFormat.get())); + if (strToDate instanceof DateV2Literal) { + DateV2Literal dateV2Literal = (DateV2Literal) strToDate; + return new DateTimeV2Literal(dateV2Literal.getYear(), dateV2Literal.getMonth(), dateV2Literal.getDay(), + 0, 0, 0); + } else if (strToDate instanceof DateTimeV2Literal) { + return (DateTimeV2Literal) strToDate; + } else { + throw new AnalysisException( + String.format("strToDate failed, stringValue: %s, dateFormat: %s", value, + dateFormat)); + } + } + } + + private DateTimeV2Literal dateAdd(DateTimeV2Literal value) throws AnalysisException { + Expression result; + switch (timeUnit) { + case "year": + result = value.plusYears(1L); + break; + case "month": + result = value.plusMonths(1L); + break; + case "day": + result = value.plusDays(1L); + break; + default: + throw new AnalysisException("MTMV partition roll up not support timeUnit: " + timeUnit); + } + if (!(result instanceof DateTimeV2Literal)) { + throw new AnalysisException("sub() should return DateTimeLiteral, result: " + result); + } + return (DateTimeV2Literal) result; + } + + private String timeToStr(DateTimeV2Literal literal, Review Comment: ```suggestion private String dateTimeToStr(DateTimeV2Literal literal, ``` ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionInfo.java: ########## @@ -17,28 +17,41 @@ package org.apache.doris.mtmv; +import org.apache.doris.analysis.Expr; +import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSortedSet; import com.google.gson.annotations.SerializedName; +import java.util.List; + /** * MTMVPartitionInfo */ public class MTMVPartitionInfo { public enum MTMVPartitionType { FOLLOW_BASE_TABLE, + EXPR, Review Comment: ROLL_UP is better? ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java: ########## Review Comment: should we change all mtmv to async mv in error message? ########## fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionExprDateTrunc.java: ########## @@ -0,0 +1,224 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv; + +import org.apache.doris.analysis.Expr; +import org.apache.doris.analysis.FunctionCallExpr; +import org.apache.doris.analysis.PartitionExprUtil; +import org.apache.doris.analysis.PartitionKeyDesc; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.PartitionType; +import org.apache.doris.catalog.Type; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.util.PropertyAnalyzer; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeArithmetic; +import org.apache.doris.nereids.trees.expressions.functions.executable.DateTimeExtractAndTransform; +import org.apache.doris.nereids.trees.expressions.literal.DateTimeV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.DateV2Literal; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; + +import com.google.common.collect.ImmutableSet; +import org.apache.commons.lang3.StringUtils; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; + +public class MTMVPartitionExprDateTrunc implements MTMVPartitionExprService { + private static Set<String> timeUnits = ImmutableSet.of("year", "month", "day"); + private String timeUnit; + + public MTMVPartitionExprDateTrunc(FunctionCallExpr functionCallExpr) throws AnalysisException { + List<Expr> paramsExprs = functionCallExpr.getParams().exprs(); + if (paramsExprs.size() != 2) { + throw new AnalysisException("date_trunc params exprs size should be 2."); + } + Expr param = paramsExprs.get(1); + if (!(param instanceof StringLiteral)) { + throw new AnalysisException("date_trunc param of time unit is not string literal."); + } + this.timeUnit = param.getStringValue().toLowerCase(); + } + + @Override + public void analyze(MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + if (!timeUnits.contains(this.timeUnit)) { + throw new AnalysisException( + String.format("timeUnit not support: %s, only support: %s", this.timeUnit, timeUnits)); + } + MTMVRelatedTableIf relatedTable = mvPartitionInfo.getRelatedTable(); + PartitionType partitionType = relatedTable.getPartitionType(); + if (partitionType == PartitionType.RANGE) { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + if (!partitionColumnType.isDateType()) { + throw new AnalysisException( + "partitionColumnType should be date/datetime " + + "when PartitionType is range and expr is date_trunc"); + } + } + } + + @Override + public String getRollUpIdentity(PartitionKeyDesc partitionKeyDesc, Map<String, String> mvProperties) + throws AnalysisException { + String res = null; + Optional<String> dateFormat = getDateFormat(mvProperties); + List<List<PartitionValue>> inValues = partitionKeyDesc.getInValues(); + for (int i = 0; i < inValues.size(); i++) { + // mtmv only support one partition column + PartitionValue partitionValue = inValues.get(i).get(0); + if (partitionValue.isNullPartition()) { + throw new AnalysisException("date trunc not support null partition value"); + } + String identity = dateTrunc(partitionValue.getStringValue(), dateFormat, false).toString(); + if (i == 0) { + res = identity; + } else { + if (!Objects.equals(res, identity)) { + throw new AnalysisException( + String.format("partition values not equal, res: %s, identity: %s", res, + identity)); + } + } + } + return res; + } + + private Optional<String> getDateFormat(Map<String, String> mvProperties) { + Optional<String> dateFormat = + StringUtils.isEmpty(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)) + ? Optional.empty() + : Optional.of(mvProperties.get(PropertyAnalyzer.PROPERTIES_PARTITION_DATE_FORMAT)); + return dateFormat; + } + + @Override + public PartitionKeyDesc generateRollUpPartitionKeyDesc(PartitionKeyDesc partitionKeyDesc, + MTMVPartitionInfo mvPartitionInfo) throws AnalysisException { + Type partitionColumnType = MTMVPartitionUtil + .getPartitionColumnType(mvPartitionInfo.getRelatedTable(), mvPartitionInfo.getRelatedCol()); + // mtmv only support one partition column + DateTimeV2Literal beginTime = dateTrunc( + partitionKeyDesc.getLowerValues().get(0).getStringValue(), + Optional.empty(), false); + + PartitionValue lowerValue = new PartitionValue(timeToStr(beginTime, partitionColumnType)); + return PartitionKeyDesc.createFixed( + Collections.singletonList(lowerValue), + Collections.singletonList( + getUpperValue(partitionKeyDesc.getUpperValues().get(0), beginTime, partitionColumnType))); + } + + private PartitionValue getUpperValue(PartitionValue upperValue, DateTimeV2Literal beginTruncTime, + Type partitionColumnType) throws AnalysisException { + if (upperValue.isMax()) { + throw new AnalysisException("date trunc not support MAXVALUE partition"); + } + // begin time and end time dateTrunc should has same result + DateTimeV2Literal endTruncTime = dateTrunc(upperValue.getStringValue(), Optional.empty(), true); + if (!Objects.equals(beginTruncTime, endTruncTime)) { + throw new AnalysisException( + String.format("partition values not equal, beginTruncTime: %s, endTruncTime: %s", beginTruncTime, + endTruncTime)); + } + DateTimeV2Literal endTime = dateAdd(beginTruncTime); + return new PartitionValue(timeToStr(endTime, partitionColumnType)); + } + + private DateTimeV2Literal dateTrunc(String value, + Optional<String> dateFormat, boolean isUpper) throws AnalysisException { + DateTimeV2Literal dateTimeLiteral = strToDate(value, dateFormat); + // for (2020-01-31,2020-02-01),if not -1, lower value and upper value will not same after rollup + if (isUpper) { + dateTimeLiteral = (DateTimeV2Literal) DateTimeArithmetic.secondsSub(dateTimeLiteral, new IntegerLiteral(1)); + } + Expression expression = DateTimeExtractAndTransform.dateTrunc(dateTimeLiteral, new VarcharLiteral(timeUnit)); + if (!(expression instanceof DateTimeV2Literal)) { + throw new AnalysisException("dateTrunc() should return DateLiteral, expression: " + expression); + } + return (DateTimeV2Literal) expression; + } + + private DateTimeV2Literal strToDate(String value, + Optional<String> dateFormat) throws AnalysisException { + try { + return new DateTimeV2Literal(value); + } catch (Exception e) { + if (!dateFormat.isPresent()) { + throw e; + } + Expression strToDate = DateTimeExtractAndTransform + .strToDate(new VarcharLiteral(value), + new VarcharLiteral(dateFormat.get())); + if (strToDate instanceof DateV2Literal) { + DateV2Literal dateV2Literal = (DateV2Literal) strToDate; + return new DateTimeV2Literal(dateV2Literal.getYear(), dateV2Literal.getMonth(), dateV2Literal.getDay(), + 0, 0, 0); + } else if (strToDate instanceof DateTimeV2Literal) { + return (DateTimeV2Literal) strToDate; + } else { + throw new AnalysisException( + String.format("strToDate failed, stringValue: %s, dateFormat: %s", value, + dateFormat)); + } + } + } + + private DateTimeV2Literal dateAdd(DateTimeV2Literal value) throws AnalysisException { + Expression result; + switch (timeUnit) { + case "year": + result = value.plusYears(1L); + break; + case "month": + result = value.plusMonths(1L); + break; + case "day": + result = value.plusDays(1L); + break; + default: + throw new AnalysisException("MTMV partition roll up not support timeUnit: " + timeUnit); + } + if (!(result instanceof DateTimeV2Literal)) { + throw new AnalysisException("sub() should return DateTimeLiteral, result: " + result); + } + return (DateTimeV2Literal) result; + } + + private String timeToStr(DateTimeV2Literal literal, + Type partitionColumnType) throws AnalysisException { + if (partitionColumnType.isDate() || partitionColumnType.isDateV2()) { + return String.format(PartitionExprUtil.DATE_FORMATTER, literal.getYear(), literal.getMonth(), + literal.getDay()); + } else if (partitionColumnType.isDatetime() || partitionColumnType.isDatetimeV2()) { + return String.format(PartitionExprUtil.DATETIME_FORMATTER, + literal.getYear(), literal.getMonth(), literal.getDay(), + literal.getHour(), literal.getMinute(), literal.getSecond()); + } else { + throw new AnalysisException( + "MTMV swnot support partition with column type : " + partitionColumnType.toString()); Review Comment: ```suggestion "MTMV swnot support partition with column type : " + partitionColumnType.toString()); ``` ```suggestion "MTMV not support partition with column type : " + partitionColumnType.toString()); ``` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For queries about this service, please contact Infrastructure at: us...@infra.apache.org --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org
