This is an automated email from the ASF dual-hosted git repository. englefly pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 52ac59c761e [Feat](Nereids) Support use mv hint (#45855) 52ac59c761e is described below commit 52ac59c761e9da7e4eefa2216abfcf2a5b8a6f05 Author: LiBinfeng <libinf...@selectdb.com> AuthorDate: Mon Dec 30 15:07:42 2024 +0800 [Feat](Nereids) Support use mv hint (#45855) ### What problem does this PR solve? support use_mv and no_use_mv hint to control mv chose strategy in cbo ```explain select /*+ use_mv(mv1)*/ * from t1;``` --- .../antlr4/org/apache/doris/nereids/DorisLexer.g4 | 2 + .../antlr4/org/apache/doris/nereids/DorisParser.g4 | 1 + .../java/org/apache/doris/catalog/OlapTable.java | 89 ++++------ .../org/apache/doris/nereids/StatementContext.java | 18 ++ .../org/apache/doris/nereids/cost/CostModelV1.java | 27 +++ .../org/apache/doris/nereids/hint/UseMvHint.java | 163 ++++++++++++------ .../doris/nereids/parser/LogicalPlanBuilder.java | 35 ++-- .../doris/nereids/properties/SelectHintUseMv.java | 10 +- .../rules/analysis/EliminateLogicalSelectHint.java | 9 +- .../mv/InitMaterializationContextHook.java | 65 ++++++- .../org/apache/doris/regression/suite/Suite.groovy | 14 ++ .../suites/nereids_p0/hint/test_use_mv.groovy | 191 +++++++++++++++++++-- 12 files changed, 463 insertions(+), 161 deletions(-) diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 index 4e94dc553cd..40a576c250f 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4 @@ -364,6 +364,7 @@ NEVER: 'NEVER'; NEXT: 'NEXT'; NGRAM_BF: 'NGRAM_BF'; NO: 'NO'; +NO_USE_MV: 'NO_USE_MV'; NON_NULLABLE: 'NON_NULLABLE'; NOT: 'NOT'; NULL: 'NULL'; @@ -538,6 +539,7 @@ UP: 'UP'; UPDATE: 'UPDATE'; USE: 'USE'; USER: 'USER'; +USE_MV: 'USE_MV'; USING: 'USING'; VALUE: 'VALUE'; VALUES: 'VALUES'; diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 12f266d48b0..6eedeaad211 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1243,6 +1243,7 @@ selectHint: hintStatements+=hintStatement (COMMA? hintStatements+=hintStatement) hintStatement : hintName=identifier (LEFT_PAREN parameters+=hintAssignment (COMMA? parameters+=hintAssignment)* RIGHT_PAREN)? + | (USE_MV | NO_USE_MV) (LEFT_PAREN tableList+=multipartIdentifier (COMMA tableList+=multipartIdentifier)* RIGHT_PAREN)? ; hintAssignment diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 32a4fd1517b..3e8964326c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -568,79 +568,66 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc } public Long getBestMvIdWithHint(List<Long> orderedMvs) { - Optional<UseMvHint> useMvHint = getUseMvHint("USE_MV"); - Optional<UseMvHint> noUseMvHint = getUseMvHint("NO_USE_MV"); + Optional<UseMvHint> useMvHint = ConnectContext.get().getStatementContext().getUseMvHint("USE_MV"); + Optional<UseMvHint> noUseMvHint = ConnectContext.get().getStatementContext().getUseMvHint("NO_USE_MV"); + List<String> names = new ArrayList<>(); + InternalCatalog catalog = Env.getCurrentEnv().getInternalCatalog(); + names.add(catalog.getName()); + names.add(getDBName()); + names.add(this.name); if (useMvHint.isPresent() && noUseMvHint.isPresent()) { - if (noUseMvHint.get().getNoUseMVName(this.name).contains(useMvHint.get().getUseMvName(this.name))) { - String errorMsg = "conflict mv exist in use_mv and no_use_mv in the same time" - + useMvHint.get().getUseMvName(this.name); - useMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR); - useMvHint.get().setErrorMessage(errorMsg); - noUseMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR); - noUseMvHint.get().setErrorMessage(errorMsg); - } - return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs); + return getMvIdWithUseMvHint(useMvHint.get(), names, orderedMvs); } else if (useMvHint.isPresent()) { - return getMvIdWithUseMvHint(useMvHint.get(), orderedMvs); + return getMvIdWithUseMvHint(useMvHint.get(), names, orderedMvs); } else if (noUseMvHint.isPresent()) { - return getMvIdWithNoUseMvHint(noUseMvHint.get(), orderedMvs); + return getMvIdWithNoUseMvHint(noUseMvHint.get(), names, orderedMvs); } return orderedMvs.get(0); } - private Long getMvIdWithUseMvHint(UseMvHint useMvHint, List<Long> orderedMvs) { + private Long getMvIdWithUseMvHint(UseMvHint useMvHint, List<String> names, List<Long> orderedMvs) { if (useMvHint.isAllMv()) { useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: " + this.name); return orderedMvs.get(0); } else { - String mvName = useMvHint.getUseMvName(this.name); - if (mvName != null) { - if (mvName.equals("`*`")) { - useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); - useMvHint.setErrorMessage("use_mv hint should only have one mv in one table: " - + this.name); - return orderedMvs.get(0); - } - Long choosedIndexId = indexNameToId.get(mvName); - if (orderedMvs.contains(choosedIndexId)) { - useMvHint.setStatus(Hint.HintStatus.SUCCESS); - return choosedIndexId; - } else { - useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); - useMvHint.setErrorMessage("do not have mv: " + mvName + " in table: " + this.name); + for (Map.Entry<String, Long> entry : indexNameToId.entrySet()) { + String mvName = entry.getKey(); + names.add(mvName); + if (useMvHint.getUseMvTableColumnMap().containsKey(names)) { + useMvHint.getUseMvTableColumnMap().put(names, true); + Long choosedIndexId = indexNameToId.get(mvName); + if (orderedMvs.contains(choosedIndexId)) { + useMvHint.setStatus(Hint.HintStatus.SUCCESS); + return choosedIndexId; + } else { + useMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); + useMvHint.setErrorMessage("do not have mv: " + mvName + " in table: " + this.name); + } } } } return orderedMvs.get(0); } - private Long getMvIdWithNoUseMvHint(UseMvHint noUseMvHint, List<Long> orderedMvs) { + private Long getMvIdWithNoUseMvHint(UseMvHint noUseMvHint, List<String> names, List<Long> orderedMvs) { if (noUseMvHint.isAllMv()) { noUseMvHint.setStatus(Hint.HintStatus.SUCCESS); return getBaseIndex().getId(); } else { - List<String> mvNames = noUseMvHint.getNoUseMVName(this.name); Set<Long> forbiddenIndexIds = Sets.newHashSet(); - for (int i = 0; i < mvNames.size(); i++) { - if (mvNames.get(i).equals("`*`")) { - noUseMvHint.setStatus(Hint.HintStatus.SUCCESS); - return getBaseIndex().getId(); - } - if (hasMaterializedIndex(mvNames.get(i))) { - Long forbiddenIndexId = indexNameToId.get(mvNames.get(i)); + for (Map.Entry<String, Long> entry : indexNameToId.entrySet()) { + String mvName = entry.getKey(); + names.add(mvName); + if (noUseMvHint.getNoUseMvTableColumnMap().containsKey(names)) { + noUseMvHint.getNoUseMvTableColumnMap().put(names, true); + Long forbiddenIndexId = indexNameToId.get(mvName); forbiddenIndexIds.add(forbiddenIndexId); - } else { - noUseMvHint.setStatus(Hint.HintStatus.SYNTAX_ERROR); - noUseMvHint.setErrorMessage("do not have mv: " + mvNames.get(i) + " in table: " + this.name); - break; } } for (int i = 0; i < orderedMvs.size(); i++) { - if (forbiddenIndexIds.contains(orderedMvs.get(i))) { - noUseMvHint.setStatus(Hint.HintStatus.SUCCESS); - } else { + if (!forbiddenIndexIds.contains(orderedMvs.get(i))) { return orderedMvs.get(i); } } @@ -648,18 +635,6 @@ public class OlapTable extends Table implements MTMVRelatedTableIf, GsonPostProc return orderedMvs.get(0); } - private Optional<UseMvHint> getUseMvHint(String useMvName) { - for (Hint hint : ConnectContext.get().getStatementContext().getHints()) { - if (hint.isSyntaxError()) { - continue; - } - if (hint.getHintName().equalsIgnoreCase(useMvName)) { - return Optional.of((UseMvHint) hint); - } - } - return Optional.empty(); - } - public List<MaterializedIndex> getVisibleIndex() { Optional<Partition> partition = idToPartition.values().stream().findFirst(); if (!partition.isPresent()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java index 41a6ccd98a9..0b671ebdb71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/StatementContext.java @@ -30,6 +30,7 @@ import org.apache.doris.datasource.mvcc.MvccTable; import org.apache.doris.datasource.mvcc.MvccTableInfo; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.hint.Hint; +import org.apache.doris.nereids.hint.UseMvHint; import org.apache.doris.nereids.memo.Group; import org.apache.doris.nereids.rules.analysis.ColumnAliasGenerator; import org.apache.doris.nereids.trees.expressions.CTEId; @@ -527,6 +528,23 @@ public class StatementContext implements Closeable { } } + /** + * get used mv hint by hint name + * @param useMvName hint name, can either be USE_MV or NO_USE_MV + * @return optional of useMvHint + */ + public Optional<UseMvHint> getUseMvHint(String useMvName) { + for (Hint hint : getHints()) { + if (hint.isSyntaxError()) { + continue; + } + if (hint.getHintName().equalsIgnoreCase(useMvName)) { + return Optional.of((UseMvHint) hint); + } + } + return Optional.empty(); + } + public Optional<Statistics> getStatistics(Id id) { if (id instanceof RelationId) { return Optional.ofNullable(this.relationIdToStatisticsMap.get((RelationId) id)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java index f6cf30d855f..32a09bb02d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/cost/CostModelV1.java @@ -19,8 +19,11 @@ package org.apache.doris.nereids.cost; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.OlapTable; import org.apache.doris.nereids.PlanContext; +import org.apache.doris.nereids.hint.Hint; +import org.apache.doris.nereids.hint.UseMvHint; import org.apache.doris.nereids.processor.post.RuntimeFilterGenerator; import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.DistributionSpecGather; @@ -63,8 +66,10 @@ import org.apache.doris.statistics.Statistics; import com.google.common.base.Preconditions; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Optional; import java.util.Set; class CostModelV1 extends PlanVisitor<Cost, PlanContext> { @@ -106,11 +111,33 @@ class CostModelV1 extends PlanVisitor<Cost, PlanContext> { double rows = statistics.getRowCount(); double aggMvBonus = 0.0; if (table.getBaseIndexId() != physicalOlapScan.getSelectedIndexId()) { + Optional<UseMvHint> useMvHint = ConnectContext.get().getStatementContext().getUseMvHint("USE_MV"); + if (useMvHint.isPresent()) { + List<String> mvQualifier = new ArrayList<>(); + for (String qualifier : table.getFullQualifiers()) { + mvQualifier.add(qualifier); + } + mvQualifier.add(table.getIndexNameById(physicalOlapScan.getSelectedIndexId())); + if (useMvHint.get().getUseMvTableColumnMap().containsKey(mvQualifier)) { + useMvHint.get().getUseMvTableColumnMap().put(mvQualifier, true); + useMvHint.get().setStatus(Hint.HintStatus.SUCCESS); + return CostV1.ofCpu(context.getSessionVariable(), Double.NEGATIVE_INFINITY); + } + } if (table.getIndexMetaByIndexId(physicalOlapScan.getSelectedIndexId()) .getKeysType().equals(KeysType.AGG_KEYS)) { aggMvBonus = rows > 1.0 ? 1.0 : rows * 0.5; } } + if (table instanceof MTMV) { + Optional<UseMvHint> useMvHint = ConnectContext.get().getStatementContext().getUseMvHint("USE_MV"); + if (useMvHint.isPresent() && useMvHint.get().getUseMvTableColumnMap() + .containsKey(table.getFullQualifiers())) { + useMvHint.get().getUseMvTableColumnMap().put(table.getFullQualifiers(), true); + useMvHint.get().setStatus(Hint.HintStatus.SUCCESS); + return CostV1.ofCpu(context.getSessionVariable(), Double.NEGATIVE_INFINITY); + } + } return CostV1.ofCpu(context.getSessionVariable(), rows - aggMvBonus); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java index 5e37bdc2760..f25d5414d07 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseMvHint.java @@ -17,10 +17,16 @@ package org.apache.doris.nereids.hint; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.base.Strings; + import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; /** * rule hint. @@ -31,74 +37,127 @@ public class UseMvHint extends Hint { private final boolean isAllMv; - private final List<String> parameters; + private final List<List<String>> tables; - private final Map<String, String> useMvTableColumnMap; + private Map<List<String>, Boolean> useMvTableColumnMap = new HashMap<>(); - private final Map<String, List<String>> noUseMvTableColumnMap; + private Map<List<String>, Boolean> noUseMvTableColumnMap = new HashMap<>(); /** * constructor of use mv hint * @param hintName use mv - * @param parameters original parameters + * @param tables original parameters * @param isUseMv use_mv hint or no_use_mv hint * @param isAllMv should all mv be controlled */ - public UseMvHint(String hintName, List<String> parameters, boolean isUseMv, boolean isAllMv) { + public UseMvHint(String hintName, List<List<String>> tables, boolean isUseMv, boolean isAllMv, List<Hint> hints) { super(hintName); this.isUseMv = isUseMv; this.isAllMv = isAllMv; - this.parameters = parameters; - this.useMvTableColumnMap = initUseMvTableColumnMap(parameters); - this.noUseMvTableColumnMap = initNoUseMvTableColumnMap(parameters); + this.tables = tables; + this.useMvTableColumnMap = initMvTableColumnMap(tables, true); + this.noUseMvTableColumnMap = initMvTableColumnMap(tables, false); + checkConflicts(hints); + } + + public Map<List<String>, Boolean> getNoUseMvTableColumnMap() { + return noUseMvTableColumnMap; } - private Map<String, String> initUseMvTableColumnMap(List<String> parameters) { - Map<String, String> tempUseMvTableColumnMap = new HashMap<>(); - if (!isUseMv) { - return tempUseMvTableColumnMap; + public Map<List<String>, Boolean> getUseMvTableColumnMap() { + return useMvTableColumnMap; + } + + private void checkConflicts(List<Hint> hints) { + String otherUseMv = isUseMv ? "NO_USE_MV" : "USE_MV"; + Optional<UseMvHint> otherUseMvHint = Optional.empty(); + for (Hint hint : hints) { + if (hint.getHintName().equals(otherUseMv)) { + otherUseMvHint = Optional.of((UseMvHint) hint); + } } - if (parameters.size() % 2 == 1) { - this.setStatus(HintStatus.SYNTAX_ERROR); - this.setErrorMessage("parameter of use_mv hint must be in pairs"); - return tempUseMvTableColumnMap; + if (!otherUseMvHint.isPresent()) { + return; } - for (int i = 0; i < parameters.size(); i += 2) { - String tableName = parameters.get(i); - String columnName = parameters.get(i + 1); - if (tempUseMvTableColumnMap.containsKey(tableName)) { - this.setStatus(HintStatus.SYNTAX_ERROR); - this.setErrorMessage("use_mv hint should only have one mv in one table: " - + tableName + "." + columnName); - break; + Map<List<String>, Boolean> otherUseMvTableColumnMap = isUseMv + ? otherUseMvHint.get().getNoUseMvTableColumnMap() : otherUseMvHint.get().getUseMvTableColumnMap(); + Map<List<String>, Boolean> thisUseMvTableColumnMap = isUseMv ? useMvTableColumnMap : noUseMvTableColumnMap; + for (Map.Entry<List<String>, Boolean> entry : thisUseMvTableColumnMap.entrySet()) { + List<String> mv = entry.getKey(); + if (otherUseMvTableColumnMap.get(mv) != null) { + String errorMsg = "conflict mv exist in use_mv and no_use_mv in the same time. Mv name: " + + mv; + super.setStatus(Hint.HintStatus.SYNTAX_ERROR); + super.setErrorMessage(errorMsg); + otherUseMvHint.get().setStatus(Hint.HintStatus.SYNTAX_ERROR); + otherUseMvHint.get().setErrorMessage(errorMsg); } - tempUseMvTableColumnMap.put(tableName, columnName); } - return tempUseMvTableColumnMap; } - private Map<String, List<String>> initNoUseMvTableColumnMap(List<String> parameters) { - Map<String, List<String>> tempNoUseMvTableColumnMap = new HashMap<>(); - if (isUseMv) { - return tempNoUseMvTableColumnMap; - } - if (parameters.size() % 2 == 1) { - this.setStatus(HintStatus.SYNTAX_ERROR); - this.setErrorMessage("parameter of no_use_mv hint must be in pairs"); - return tempNoUseMvTableColumnMap; + private Map<List<String>, Boolean> initMvTableColumnMap(List<List<String>> parameters, boolean initUseMv) { + Map<List<String>, Boolean> mvTableColumnMap; + if (initUseMv && !isUseMv) { + return useMvTableColumnMap; + } else if (!initUseMv && isUseMv) { + return noUseMvTableColumnMap; + } else if (initUseMv && isUseMv) { + mvTableColumnMap = useMvTableColumnMap; + } else { + mvTableColumnMap = noUseMvTableColumnMap; } - for (int i = 0; i < parameters.size(); i += 2) { - String tableName = parameters.get(i); - String columnName = parameters.get(i + 1); - if (tempNoUseMvTableColumnMap.containsKey(tableName)) { - tempNoUseMvTableColumnMap.get(tableName).add(columnName); + for (List<String> table : parameters) { + // materialize view qualifier should have length between 1 and 4 + // which 1 and 3 represent of async materialize view, 2 and 4 represent of sync materialize view + // number of parameters meaning + // 1 async materialize view, mvName + // 2 sync materialize view, tableName.mvName + // 3 async materialize view, catalogName.dbName.mvName + // 3 sync materialize view, catalogName.dbName.tableName.mvName + if (table.size() < 1 || table.size() > 4) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("parameters number of no_use_mv hint must between 1 and 4"); + return mvTableColumnMap; + } + String mvName = table.get(table.size() - 1); + if (mvName.equals("`*`") && isUseMv) { + this.setStatus(Hint.HintStatus.SYNTAX_ERROR); + this.setErrorMessage("use_mv hint should only have one mv in one table"); + return mvTableColumnMap; + } + List<String> dbQualifier = new ArrayList<>(); + if (table.size() == 3 || table.size() == 4) { + mvTableColumnMap.put(table, false); + return mvTableColumnMap; + } + CatalogIf catalogIf = ConnectContext.get().getCurrentCatalog(); + if (catalogIf == null) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("Current catalog is not set."); + return mvTableColumnMap; + } + String catalogName = catalogIf.getName(); + String dbName = ConnectContext.get().getDatabase(); + if (Strings.isNullOrEmpty(dbName)) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("Current database is not set."); + return mvTableColumnMap; + } + dbQualifier.add(catalogName); + dbQualifier.add(dbName); + if (table.size() == 2) { + dbQualifier.add(table.get(0)); + } + dbQualifier.add(mvName); + if (mvTableColumnMap.containsKey(dbQualifier)) { + this.setStatus(HintStatus.SYNTAX_ERROR); + this.setErrorMessage("repeated parameters in use_mv hint: " + dbQualifier); + return mvTableColumnMap; } else { - List<String> list = new ArrayList<>(); - list.add(columnName); - tempNoUseMvTableColumnMap.put(tableName, list); + mvTableColumnMap.put(dbQualifier, false); } } - return tempNoUseMvTableColumnMap; + return mvTableColumnMap; } public boolean isUseMv() { @@ -109,14 +168,6 @@ public class UseMvHint extends Hint { return isAllMv; } - public String getUseMvName(String tableName) { - return useMvTableColumnMap.get(tableName); - } - - public List<String> getNoUseMVName(String tableName) { - return noUseMvTableColumnMap.get(tableName); - } - @Override public String getExplainString() { StringBuilder out = new StringBuilder(); @@ -125,14 +176,14 @@ public class UseMvHint extends Hint { } else { out.append("no_use_mv"); } - if (!parameters.isEmpty()) { + if (!tables.isEmpty()) { out.append("("); - for (int i = 0; i < parameters.size(); i++) { + for (int i = 0; i < tables.size(); i++) { if (i % 2 == 0) { - out.append(parameters.get(i)); + out.append(tables.get(i)); } else { out.append("."); - out.append(parameters.get(i)); + out.append(tables.get(i)); out.append(" "); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index b8fea4049ca..4341aafefaf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -3516,6 +3516,14 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> { return last; } + private List<List<String>> getTableList(List<MultipartIdentifierContext> ctx) { + List<List<String>> tableList = new ArrayList<>(); + for (MultipartIdentifierContext tableCtx : ctx) { + tableList.add(visitMultipartIdentifier(tableCtx)); + } + return tableList; + } + private LogicalPlan withSelectHint(LogicalPlan logicalPlan, List<ParserRuleContext> hintContexts) { if (hintContexts.isEmpty()) { return logicalPlan; @@ -3524,6 +3532,13 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> { for (ParserRuleContext hintContext : hintContexts) { SelectHintContext selectHintContext = (SelectHintContext) hintContext; for (HintStatementContext hintStatement : selectHintContext.hintStatements) { + if (hintStatement.USE_MV() != null) { + hints.add(new SelectHintUseMv("USE_MV", getTableList(hintStatement.tableList), true)); + continue; + } else if (hintStatement.NO_USE_MV() != null) { + hints.add(new SelectHintUseMv("NO_USE_MV", getTableList(hintStatement.tableList), false)); + continue; + } String hintName = hintStatement.hintName.getText().toLowerCase(Locale.ROOT); switch (hintName) { case "set_var": @@ -3579,26 +3594,6 @@ public class LogicalPlanBuilder extends DorisParserBaseVisitor<Object> { } hints.add(new SelectHintUseCboRule(hintName, noUseRuleParameters, true)); break; - case "use_mv": - List<String> useIndexParameters = new ArrayList<String>(); - for (HintAssignmentContext kv : hintStatement.parameters) { - String parameterName = visitIdentifierOrText(kv.key); - if (kv.key != null) { - useIndexParameters.add(parameterName); - } - } - hints.add(new SelectHintUseMv(hintName, useIndexParameters, true)); - break; - case "no_use_mv": - List<String> noUseIndexParameters = new ArrayList<String>(); - for (HintAssignmentContext kv : hintStatement.parameters) { - String parameterName = visitIdentifierOrText(kv.key); - if (kv.key != null) { - noUseIndexParameters.add(parameterName); - } - } - hints.add(new SelectHintUseMv(hintName, noUseIndexParameters, false)); - break; default: break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseMv.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseMv.java index 35ce25fb4f4..e943de40332 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseMv.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseMv.java @@ -23,18 +23,18 @@ import java.util.List; * select hint UseMv. */ public class SelectHintUseMv extends SelectHint { - private final List<String> parameters; + private final List<List<String>> tables; private final boolean isUseMv; - public SelectHintUseMv(String hintName, List<String> parameters, boolean isUseMv) { + public SelectHintUseMv(String hintName, List<List<String>> tables, boolean isUseMv) { super(hintName); - this.parameters = parameters; + this.tables = tables; this.isUseMv = isUseMv; } - public List<String> getParameters() { - return parameters; + public List<List<String>> getTables() { + return tables; } public boolean isUseMv() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java index 61f0bb5fb69..f7327100006 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java @@ -125,9 +125,9 @@ public class EliminateLogicalSelectHint extends OneRewriteRuleFactory { } private void extractMv(SelectHintUseMv selectHint, StatementContext statementContext) { - boolean isAllMv = selectHint.getParameters().isEmpty(); - UseMvHint useMvHint = new UseMvHint(selectHint.getHintName(), selectHint.getParameters(), - selectHint.isUseMv(), isAllMv); + boolean isAllMv = selectHint.getTables().isEmpty(); + UseMvHint useMvHint = new UseMvHint(selectHint.getHintName(), selectHint.getTables(), + selectHint.isUseMv(), isAllMv, statementContext.getHints()); for (Hint hint : statementContext.getHints()) { if (hint.getHintName().equals(selectHint.getHintName())) { hint.setStatus(Hint.HintStatus.SYNTAX_ERROR); @@ -136,9 +136,6 @@ public class EliminateLogicalSelectHint extends OneRewriteRuleFactory { useMvHint.setErrorMessage("only one " + selectHint.getHintName() + " hint is allowed"); } } - if (!useMvHint.isSyntaxError()) { - ConnectContext.get().getSessionVariable().setEnableSyncMvCostBasedRewrite(false); - } statementContext.addHint(useMvHint); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java index db270390f9b..bf21931cb86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/InitMaterializationContextHook.java @@ -32,7 +32,10 @@ import org.apache.doris.mtmv.MTMVUtil; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.PlannerHook; +import org.apache.doris.nereids.hint.Hint; +import org.apache.doris.nereids.hint.UseMvHint; import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.qe.ConnectContext; import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; @@ -90,7 +93,7 @@ public class InitMaterializationContextHook implements PlannerHook { .isEnableSyncMvCostBasedRewrite()) { for (TableIf tableIf : collectedTables) { if (tableIf instanceof OlapTable) { - for (SyncMaterializationContext context : createSyncMvContexts( + for (MaterializationContext context : createSyncMvContexts( (OlapTable) tableIf, cascadesContext)) { cascadesContext.addMaterializationContext(context); } @@ -104,6 +107,58 @@ public class InitMaterializationContextHook implements PlannerHook { } } + private List<MaterializationContext> getMvIdWithUseMvHint(List<MaterializationContext> mtmvCtxs, + UseMvHint useMvHint) { + List<MaterializationContext> hintMTMVs = new ArrayList<>(); + for (MaterializationContext mtmvCtx : mtmvCtxs) { + List<String> mvQualifier = mtmvCtx.generateMaterializationIdentifier(); + if (useMvHint.getUseMvTableColumnMap().containsKey(mvQualifier)) { + hintMTMVs.add(mtmvCtx); + } + } + return hintMTMVs; + } + + private List<MaterializationContext> getMvIdWithNoUseMvHint(List<MaterializationContext> mtmvCtxs, + UseMvHint useMvHint) { + List<MaterializationContext> hintMTMVs = new ArrayList<>(); + if (useMvHint.isAllMv()) { + useMvHint.setStatus(Hint.HintStatus.SUCCESS); + return hintMTMVs; + } + for (MaterializationContext mtmvCtx : mtmvCtxs) { + List<String> mvQualifier = mtmvCtx.generateMaterializationIdentifier(); + if (useMvHint.getNoUseMvTableColumnMap().containsKey(mvQualifier)) { + useMvHint.setStatus(Hint.HintStatus.SUCCESS); + useMvHint.getNoUseMvTableColumnMap().put(mvQualifier, true); + } else { + hintMTMVs.add(mtmvCtx); + } + } + return hintMTMVs; + } + + /** + * get mtmvs by hint + * @param mtmvCtxs input mtmvs which could be used to rewrite sql + * @return set of mtmvs which pass the check of useMvHint + */ + public List<MaterializationContext> getMaterializationContextByHint(List<MaterializationContext> mtmvCtxs) { + Optional<UseMvHint> useMvHint = ConnectContext.get().getStatementContext().getUseMvHint("USE_MV"); + Optional<UseMvHint> noUseMvHint = ConnectContext.get().getStatementContext().getUseMvHint("NO_USE_MV"); + if (!useMvHint.isPresent() && !noUseMvHint.isPresent()) { + return mtmvCtxs; + } + List<MaterializationContext> result = mtmvCtxs; + if (noUseMvHint.isPresent()) { + result = getMvIdWithNoUseMvHint(result, noUseMvHint.get()); + } + if (useMvHint.isPresent()) { + result = getMvIdWithUseMvHint(result, useMvHint.get()); + } + return result; + } + protected Set<MTMV> getAvailableMTMVs(Set<TableIf> usedTables, CascadesContext cascadesContext) { List<BaseTableInfo> usedBaseTables = usedTables.stream().map(BaseTableInfo::new).collect(Collectors.toList()); @@ -161,13 +216,13 @@ public class InitMaterializationContextHook implements PlannerHook { cascadesContext.getConnectContext().getQueryIdentifier()), e); } } - return asyncMaterializationContext; + return getMaterializationContextByHint(asyncMaterializationContext); } - private List<SyncMaterializationContext> createSyncMvContexts(OlapTable olapTable, + private List<MaterializationContext> createSyncMvContexts(OlapTable olapTable, CascadesContext cascadesContext) { int indexNumber = olapTable.getIndexNumber(); - List<SyncMaterializationContext> contexts = new ArrayList<>(indexNumber); + List<MaterializationContext> contexts = new ArrayList<>(indexNumber); long baseIndexId = olapTable.getBaseIndexId(); int keyCount = 0; for (Column column : olapTable.getFullSchema()) { @@ -215,7 +270,7 @@ public class InitMaterializationContextHook implements PlannerHook { entry.getValue(), entry.getValue()), exception); } } - return contexts; + return getMaterializationContextByHint(contexts); } private String assembleCreateMvSqlForDupOrUniqueTable(String baseTableName, String mvName, List<Column> columns) { diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy index 29f5631c4ca..4f9409308d5 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Suite.groovy @@ -2283,6 +2283,20 @@ class Suite implements GroovyInterceptable { mv_rewrite_fail(query_sql, mv_name, true) } + def async_create_mv = { db, mv_sql, mv_name -> + sql """DROP MATERIALIZED VIEW IF EXISTS ${mv_name}""" + sql""" + CREATE MATERIALIZED VIEW ${mv_name} + BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ('replication_num' = '1') + AS ${mv_sql} + """ + + def job_name = getJobName(db, mv_name); + waitingMTMVTaskFinished(job_name) + } + def token = context.config.metaServiceToken def instance_id = context.config.multiClusterInstance def get_be_metric = { ip, port, field -> diff --git a/regression-test/suites/nereids_p0/hint/test_use_mv.groovy b/regression-test/suites/nereids_p0/hint/test_use_mv.groovy index 041b42dbb67..041b3b3a7da 100644 --- a/regression-test/suites/nereids_p0/hint/test_use_mv.groovy +++ b/regression-test/suites/nereids_p0/hint/test_use_mv.groovy @@ -40,9 +40,10 @@ suite("test_use_mv") { CREATE TABLE `t1` ( `k1` int(11) NULL, `k2` int(11) NULL, + `k3` int(11) NULL, `v1` int(11) SUM NULL ) ENGINE=OLAP - AGGREGATE KEY(`k1`, `k2`) + AGGREGATE KEY(`k1`, `k2`, `k3`) COMMENT 'OLAP' DISTRIBUTED BY HASH(`k1`) BUCKETS 3 PROPERTIES ( @@ -52,42 +53,67 @@ suite("test_use_mv") { "disable_auto_compaction" = "false" ); """ + sql """ insert into t1 values (101, 101, 101, 102);""" sql """ alter table t1 add rollup r1(k2, k1); """ waitForRollUpJob("t1", 150000) sql """ alter table t1 add rollup r2(k2); """ waitForRollUpJob("t1", 150000) createMV("create materialized view k1_k2_sumk3 as select k1, k2, sum(v1) from t1 group by k1, k2;") + + sql """set ENABLE_SYNC_MV_COST_BASED_REWRITE=false;""" explain { sql """select /*+ no_use_mv */ k1 from t1;""" notContains("t1(r1)") } - explain { - sql """select /*+ no_use_mv(t1) */ k1 from t1;""" - contains("parameter of no_use_mv hint must be in pairs") - } explain { sql """select /*+ no_use_mv(t1.`*`) */ k1 from t1;""" contains("t1(t1)") } explain { - sql """select /*+ use_mv(t1.`*`) */ k1 from t1;""" - contains("use_mv hint should only have one mv in one table") + sql """select /*+ use_mv(t1.r1) use_mv(t1.r2) */ k1 from t1;""" + contains("only one USE_MV hint is allowed") + } + explain { + sql """select /*+ no_use_mv(t1.r1) no_use_mv(t1.r2) */ k1 from t1;""" + contains("only one NO_USE_MV hint is allowed") + } + explain { + sql """select /*+ no_use_mv(t1.r3) */ k1 from t1;""" + contains("no_use_mv([t1, r3])") + } + explain { + sql """select /*+ use_mv(t1.r1) no_use_mv(t1.r1) */ k1 from t1;""" + contains("conflict mv exist in use_mv and no_use_mv in the same time") + } + explain { + sql """select /*+ use_mv(t1.k1_k2_sumk3) */ k1, k2, sum(v1) from t1 group by k1, k2;""" + contains("t1(k1_k2_sumk3)") + } + explain { + sql """select /*+ use_mv(t1.k1_k2_sumk3) */ k1, k2, min(v1) from t1 group by k1, k2;""" + notContains("t1(k1_k2_sumk3)") + } + + sql """set ENABLE_SYNC_MV_COST_BASED_REWRITE=true;""" + explain { + sql """select /*+ no_use_mv(t1.*) */ k1 from t1 group by k1;""" + notContains("t1(r1)") } explain { - sql """select /*+ use_mv(t1.r1,t1.r2) */ k1 from t1;""" - contains("use_mv hint should only have one mv in one table") + sql """select /*+ no_use_mv(t1.`*`) */ k1 from t1;""" + contains("t1(t1)") } explain { sql """select /*+ use_mv(t1.r1) use_mv(t1.r2) */ k1 from t1;""" - contains("one use_mv hint is allowed") + contains("only one USE_MV hint is allowed") } explain { sql """select /*+ no_use_mv(t1.r1) no_use_mv(t1.r2) */ k1 from t1;""" - contains("only one no_use_mv hint is allowed") + contains("only one NO_USE_MV hint is allowed") } explain { sql """select /*+ no_use_mv(t1.r3) */ k1 from t1;""" - contains("do not have mv: r3 in table: t1") + contains("UnUsed: no_use_mv([t1, r3])") } explain { sql """select /*+ use_mv(t1.r1) no_use_mv(t1.r1) */ k1 from t1;""" @@ -102,4 +128,145 @@ suite("test_use_mv") { notContains("t1(k1_k2_sumk3)") } + // create database and tables + def db = "test_cbo_use_mv" + sql "DROP DATABASE IF EXISTS ${db}" + sql "CREATE DATABASE IF NOT EXISTS ${db}" + sql "use ${db}" + + sql """drop table if exists t1;""" + sql """drop table if exists t2;""" + sql """drop table if exists t3;""" + + sql """create table t1 (c1 int, c11 int) distributed by hash(c1) buckets 3 properties('replication_num' = '1');""" + sql """create table t2 (c2 int, c22 int) distributed by hash(c2) buckets 3 properties('replication_num' = '1');""" + sql """create table t3 (c3 int, c33 int) distributed by hash(c3) buckets 3 properties('replication_num' = '1');""" + + sql """insert into t1 values (101, 101);""" + sql """insert into t2 values (102, 102);""" + sql """insert into t3 values (103, 103);""" + + def query1 = """select * from t1""" + def query2 = """select * from t2""" + def query3 = """select c1 from t1""" + def query4 = """select c3 from t3""" + + async_create_mv(db, query1, "mv1") + async_create_mv(db, query2, "mv2") + async_create_mv(db, query3, "mv3") + async_create_mv(db, query4, "mv4") + + sql """use ${db};""" + explain { + sql """memo plan select /*+ use_mv(mv1)*/ * from t1;""" + contains("internal.test_cbo_use_mv.mv1 chose") + } + explain { + sql """memo plan select /*+ no_use_mv(mv1)*/ * from t1;""" + contains("Used: no_use_mv([mv1])") + notContains("internal.test_cbo_use_mv.mv1 chose") + } + sql """use test_use_mv""" + explain { + sql """memo plan select /*+ use_mv(mv1)*/ * from ${db}.t1;""" + contains("UnUsed: use_mv([mv1])") + } + explain { + sql """memo plan select /*+ no_use_mv(mv1)*/ * from ${db}.t1;""" + contains("UnUsed: no_use_mv([mv1])") + } + sql """use ${db};""" + explain { + sql """memo plan select /*+ use_mv(internal.${db}.mv1)*/ * from t1;""" + contains("Used: use_mv([internal, test_cbo_use_mv, mv1])") + contains("internal.test_cbo_use_mv.mv1 chose") + } + explain { + sql """memo plan select /*+ no_use_mv(internal.${db}.mv1)*/ * from t1;""" + contains("Used: no_use_mv([internal, test_cbo_use_mv, mv1])") + } + sql """use ${db};""" + explain { + sql """memo plan select /*+ use_mv(mv1) */ * from t1""" + contains("internal.test_cbo_use_mv.mv1 chose") + contains("Used: use_mv([mv1])") + } + explain { + sql """memo plan select /*+ no_use_mv(mv1) */ * from t1""" + contains("Used: no_use_mv([mv1])") + notContains("internal.test_cbo_use_mv.mv1 chose") + } + explain { + sql """memo plan select /*+ use_mv(mv4) */ * from t3""" + contains("UnUsed: use_mv([mv4])") + } + explain { + sql """memo plan select /*+ no_use_mv(mv4) */ * from t3""" + contains("Used: no_use_mv([mv4])") + } + explain { + sql """memo plan select /*+ use_mv(mv1, mv2) */ * from t1 union all select * from t2""" + contains("Used: use_mv([mv1].[mv2] )") + contains("internal.test_cbo_use_mv.mv2 chose") + contains("internal.test_cbo_use_mv.mv1 chose") + } + explain { + sql """memo plan select /*+ no_use_mv(mv1, mv2) */ * from t1 union all select * from t2""" + contains("Used: no_use_mv([mv1].[mv2] )") + notContains("internal.test_cbo_use_mv.mv2 chose") + notContains("internal.test_cbo_use_mv.mv1 chose") + } + explain { + sql """memo plan select /*+ use_mv(mv1) no_use_mv(mv2) */ * from t1 union all select * from t2""" + contains("Used: use_mv([mv1]) no_use_mv([mv2])") + notContains("internal.test_cbo_use_mv.mv2 chose") + contains("internal.test_cbo_use_mv.mv1 chose") + } + explain { + sql """memo plan select /*+ use_mv(mv2) no_use_mv(mv1) */ * from t1 union all select * from t2""" + contains("Used: use_mv([mv2]) no_use_mv([mv1])") + notContains("internal.test_cbo_use_mv.mv1 chose") + contains("internal.test_cbo_use_mv.mv2 chose") + } + explain { + sql """memo plan select /*+ use_mv(mv1, mv3) */ c1 from t1""" + contains("Used: use_mv([mv1].[mv3] )") + contains("internal.test_cbo_use_mv.mv1 chose") + contains("internal.test_cbo_use_mv.mv3 not chose") + } + explain { + sql """memo plan select /*+ use_mv(mv3, mv1) */ c1 from t1""" + contains("Used: use_mv([mv3].[mv1] )") + contains("internal.test_cbo_use_mv.mv1 chose") + contains("internal.test_cbo_use_mv.mv3 not chose") + } + explain { + sql """memo plan select /*+ use_mv(mv1) */ c1 from t1""" + contains("Used: use_mv([mv1])") + contains("internal.test_cbo_use_mv.mv1 chose") + } + explain { + sql """memo plan select /*+ use_mv(mv3) */ c1 from t1""" + contains("Used: use_mv([mv3])") + contains("internal.test_cbo_use_mv.mv3 chose") + } + explain { + sql """memo plan select /*+ no_use_mv(mv1, mv3) */ c1 from t1""" + contains("Used: no_use_mv([mv1].[mv3] )") + notContains("internal.test_cbo_use_mv.mv3") + notContains("internal.test_cbo_use_mv.mv1") + } + explain { + sql """memo plan select /*+ no_use_mv(mv1) */ c1 from t1""" + contains("Used: no_use_mv([mv1])") + contains("internal.test_cbo_use_mv.mv3 chose") + notContains("internal.test_cbo_use_mv.mv1") + } + explain { + sql """memo plan select /*+ no_use_mv(mv3) */ c1 from t1""" + contains("Used: no_use_mv([mv3])") + contains("internal.test_cbo_use_mv.mv1 chose") + notContains("internal.test_cbo_use_mv.mv3") + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org