This is an automated email from the ASF dual-hosted git repository. morrysnow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push: new 2745e044e54 [fix](mtmv) Fix enable_materialized_view_nest_rewrite session variable is useless in some scene (#41472) 2745e044e54 is described below commit 2745e044e54f9e552585f6f1fc2075969fa66d6d Author: seawinde <149132972+seawi...@users.noreply.github.com> AuthorDate: Fri Oct 11 16:07:28 2024 +0800 [fix](mtmv) Fix enable_materialized_view_nest_rewrite session variable is useless in some scene (#41472) This is brought by #34050 if set `enable_materialized_view_nest_rewrite = false`, as expected, top level materialized view should rewritten fail, but now successfully. Such as first level materialized view def is CREATE MATERIALIZED VIEW level1 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT l_orderkey, l_linenumber, l_partkey, o_orderkey, o_custkey FROM lineitem_2 INNER JOIN orders_2 ON l_orderkey = o_orderkey; second level materialized view def is CREATE MATERIALIZED VIEW level2 BUILD IMMEDIATE REFRESH COMPLETE ON MANUAL DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT l_orderkey, l_linenumber, o_orderkey, sum(l_partkey) AS total_revenue, max(o_custkey) AS max_discount FROM join_mv1 GROUP BY l_orderkey, l_linenumber, o_orderkey; if set `enable_materialized_view_nest_rewrite = false`, only `level1` can rewriten succesfully and chosen by cbo if set `enable_materialized_view_nest_rewrite = true`, both `level1` and `level2` can rewriten succesfully and `level2` should be chosen by cbo. This pr fixed this --- .../org/apache/doris/nereids/CascadesContext.java | 9 ++ .../apache/doris/nereids/memo/StructInfoMap.java | 16 +- .../org/apache/doris/nereids/rules/RuleSet.java | 2 - .../mv/AbstractMaterializedViewRule.java | 13 +- .../mv/MaterializedViewOnlyJoinRule.java | 45 ------ .../mv/MaterializedViewProjectFilterJoinRule.java | 1 + .../exploration/mv/MaterializedViewUtils.java | 2 +- .../mv/direct_query/direct_query.groovy | 1 + .../nested_mtmv_rewrite_switch.groovy | 169 +++++++++++++++++++++ 9 files changed, 207 insertions(+), 51 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index 68812245134..25767134d4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -122,6 +122,7 @@ public class CascadesContext implements ScheduleContext { private final Optional<CascadesContext> parent; private final Set<MaterializationContext> materializationContexts; + private final Set<List<String>> materializationRewrittenSuccessSet = new HashSet<>(); private boolean isLeadingJoin = false; private boolean isLeadingDisableJoinReorder = false; @@ -366,6 +367,14 @@ public class CascadesContext implements ScheduleContext { this.materializationContexts.add(materializationContext); } + public Set<List<String>> getMaterializationRewrittenSuccessSet() { + return materializationRewrittenSuccessSet; + } + + public void addMaterializationRewrittenSuccess(List<String> materializationQualifier) { + this.materializationRewrittenSuccessSet.add(materializationQualifier); + } + /** * getAndCacheSessionVariable */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java index 0c11b5fbb22..4aa4f146b87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/StructInfoMap.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.memo; +import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Pair; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.exploration.mv.StructInfo; @@ -126,6 +127,9 @@ public class StructInfoMap { List<Set<BitSet>> childrenTableMap = new LinkedList<>(); if (groupExpression.children().isEmpty()) { BitSet leaf = constructLeaf(groupExpression, cascadesContext); + if (leaf.isEmpty()) { + break; + } groupExpressionMap.put(leaf, Pair.of(groupExpression, new LinkedList<>())); continue; } @@ -163,9 +167,19 @@ public class StructInfoMap { private BitSet constructLeaf(GroupExpression groupExpression, CascadesContext cascadesContext) { Plan plan = groupExpression.getPlan(); BitSet tableMap = new BitSet(); + boolean enableMaterializedViewNestRewrite = cascadesContext.getConnectContext().getSessionVariable() + .isEnableMaterializedViewNestRewrite(); if (plan instanceof LogicalCatalogRelation) { + TableIf table = ((LogicalCatalogRelation) plan).getTable(); + // If disable materialized view nest rewrite, and mv already rewritten successfully once, doesn't construct + // table id map for nest mv rewrite + if (!enableMaterializedViewNestRewrite + && cascadesContext.getMaterializationRewrittenSuccessSet().contains(table.getFullQualifiers())) { + return tableMap; + + } tableMap.set(cascadesContext.getStatementContext() - .getTableId(((LogicalCatalogRelation) plan).getTable()).asInt()); + .getTableId(table).asInt()); } // one row relation / CTE consumer return tableMap; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java index 49613711844..bcd12ac17d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java @@ -39,7 +39,6 @@ import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProje import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterProjectScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewFilterScanRule; -import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyJoinRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewOnlyScanRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectAggregateRule; import org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectFilterAggregateRule; @@ -237,7 +236,6 @@ public class RuleSet { .build(); public static final List<Rule> MATERIALIZED_VIEW_RULES = planRuleFactories() - .add(MaterializedViewOnlyJoinRule.INSTANCE) .add(MaterializedViewProjectJoinRule.INSTANCE) .add(MaterializedViewFilterJoinRule.INSTANCE) .add(MaterializedViewFilterProjectJoinRule.INSTANCE) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java index 7d84b8ab36b..dd17d754244 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java @@ -37,6 +37,7 @@ import org.apache.doris.nereids.rules.exploration.mv.StructInfo.PartitionRemover import org.apache.doris.nereids.rules.exploration.mv.mapping.ExpressionMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.RelationMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; +import org.apache.doris.nereids.rules.rewrite.MergeProjects; import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -354,6 +355,13 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac rewrittenPlanOutput, queryPlan.getOutput())); continue; } + // Merge project + rewrittenPlan = MaterializedViewUtils.rewriteByRules(cascadesContext, + childContext -> { + Rewriter.getCteChildrenRewriter(childContext, + ImmutableList.of(Rewriter.bottomUp(new MergeProjects()))).execute(); + return childContext.getRewritePlan(); + }, rewrittenPlan, queryPlan); if (!isOutputValid(queryPlan, rewrittenPlan)) { LogicalProperties logicalProperties = rewrittenPlan.getLogicalProperties(); materializationContext.recordFailReason(queryStructInfo, @@ -363,7 +371,7 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac logicalProperties, queryPlan.getLogicalProperties())); continue; } - recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext); + recordIfRewritten(queryStructInfo.getOriginalPlan(), materializationContext, cascadesContext); trySetStatistics(materializationContext, cascadesContext); rewriteResults.add(rewrittenPlan); // if rewrite successfully, try to regenerate mv scan because it maybe used again @@ -852,8 +860,9 @@ public abstract class AbstractMaterializedViewRule implements ExplorationRuleFac return checkQueryPattern(structInfo, cascadesContext); } - protected void recordIfRewritten(Plan plan, MaterializationContext context) { + protected void recordIfRewritten(Plan plan, MaterializationContext context, CascadesContext cascadesContext) { context.setSuccess(true); + cascadesContext.addMaterializationRewrittenSuccess(context.generateMaterializationIdentifier()); if (plan.getGroupExpression().isPresent()) { context.addMatchedGroup(plan.getGroupExpression().get().getOwnerGroup().getGroupId(), true); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java deleted file mode 100644 index 2735ca87fe9..00000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewOnlyJoinRule.java +++ /dev/null @@ -1,45 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.exploration.mv; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.Plan; -import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; -import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; - -import com.google.common.collect.ImmutableList; - -import java.util.List; - -/** - * This is responsible for join pattern such as only join - */ -public class MaterializedViewOnlyJoinRule extends AbstractMaterializedViewJoinRule { - - public static final MaterializedViewOnlyJoinRule INSTANCE = new MaterializedViewOnlyJoinRule(); - - @Override - public List<Rule> buildRules() { - return ImmutableList.of(logicalJoin(any().when(LogicalPlan.class::isInstance), - any().when(LogicalPlan.class::isInstance)).thenApplyMultiNoThrow(ctx -> { - LogicalJoin<Plan, Plan> root = ctx.root; - return rewrite(root, ctx.cascadesContext); - }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_JOIN)); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java index d82f838ea6b..05f54ac3401 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterJoinRule.java @@ -31,6 +31,7 @@ import java.util.List; /** * This is responsible for join pattern such as project on filter on join + * Needed because variant data type would have filter on join directly, such as query query3_5 in variant_mv.groovy */ public class MaterializedViewProjectFilterJoinRule extends AbstractMaterializedViewJoinRule { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 342c88ff677..7bad7f6db28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -212,8 +212,8 @@ public class MaterializedViewUtils { structInfosBuilder.add(structInfo); } } - return structInfosBuilder.build(); } + return structInfosBuilder.build(); } // if plan doesn't belong to any group, construct it directly return ImmutableList.of(StructInfo.of(plan, originalPlan, cascadesContext)); diff --git a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy index 404154300d2..7b45349512b 100644 --- a/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy +++ b/regression-test/suites/nereids_rules_p0/mv/direct_query/direct_query.groovy @@ -20,6 +20,7 @@ suite("direct_query_mv") { String db = context.config.getDbNameByFile(context.file) sql "use ${db}" sql "set runtime_filter_mode=OFF" + sql """set enable_materialized_view_nest_rewrite = true; """ sql """ drop table if exists orders diff --git a/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy new file mode 100644 index 00000000000..f9a84fa5250 --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/nested_mtmv_switch/nested_mtmv_rewrite_switch.groovy @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("nested_mtmv_rewrite_switch") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "SET enable_materialized_view_rewrite=true" + + sql """ + drop table if exists orders_2 + """ + sql """ + CREATE TABLE `orders_2` ( + `o_orderkey` BIGINT, + `o_custkey` int, + `o_orderstatus` VARCHAR(1), + `o_totalprice` DECIMAL(15, 2), + `o_orderpriority` VARCHAR(15), + `o_clerk` VARCHAR(15), + `o_shippriority` int, + `o_comment` VARCHAR(79), + `o_orderdate` DATE + ) ENGINE=olap + + PROPERTIES ( + "replication_num" = "1" + + ); + """ + + sql """ + drop table if exists lineitem_2 + """ + sql """ + CREATE TABLE `lineitem_2` ( + `l_orderkey` BIGINT, + `l_linenumber` INT, + `l_partkey` INT, + `l_suppkey` INT, + `l_quantity` DECIMAL(15, 2), + `l_extendedprice` DECIMAL(15, 2), + `l_discount` DECIMAL(15, 2), + `l_tax` DECIMAL(15, 2), + `l_returnflag` VARCHAR(1), + `l_linestatus` VARCHAR(1), + `l_commitdate` DATE, + `l_receiptdate` DATE, + `l_shipinstruct` VARCHAR(25), + `l_shipmode` VARCHAR(10), + `l_comment` VARCHAR(44), + `l_shipdate` DATE + ) ENGINE=olap + + PROPERTIES ( + "replication_num" = "1" + + ); + """ + + sql """ + insert into orders_2 values + (null, 1, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (1, null, 'o', 109.2, 'c','d',2, 'mm', '2023-10-17'), + (3, 3, null, 99.5, 'a', 'b', 1, 'yy', '2023-10-19'), + (1, 2, 'o', null, 'a', 'b', 1, 'yy', '2023-10-20'), + (2, 3, 'k', 109.2, null,'d',2, 'mm', '2023-10-21'), + (3, 1, 'k', 99.5, 'a', null, 1, 'yy', '2023-10-22'), + (1, 3, 'o', 99.5, 'a', 'b', null, 'yy', '2023-10-19'), + (2, 1, 'o', 109.2, 'c','d',2, null, '2023-10-18'), + (3, 2, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-17'), + (4, 5, 'k', 99.5, 'a', 'b', 1, 'yy', '2023-10-19'); + """ + + sql""" + insert into lineitem_2 values + (null, 1, 2, 3, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (1, null, 3, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-18', '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (3, 3, null, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', '2023-10-19', 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 2, 3, null, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'), + (2, 3, 2, 1, 5.5, 6.5, 7.5, 8.5, 'o', 'k', null, '2023-10-18', 'a', 'b', 'yyyyyyyyy', '2023-10-18'), + (3, 1, 1, 2, 7.5, 8.5, 9.5, 10.5, 'k', 'o', '2023-10-19', null, 'c', 'd', 'xxxxxxxxx', '2023-10-19'), + (1, 3, 2, 2, 5.5, 6.5, 7.5, 8.5, 'o', 'k', '2023-10-17', '2023-10-17', 'a', 'b', 'yyyyyyyyy', '2023-10-17'); + """ + + sql """analyze table orders_2 with sync;""" + sql """analyze table lineitem_2 with sync;""" + + + def compare_res = { def stmt -> + sql "SET enable_materialized_view_rewrite=false" + def origin_res = sql stmt + logger.info("origin_res: " + origin_res) + sql "SET enable_materialized_view_rewrite=true" + def mv_origin_res = sql stmt + logger.info("mv_origin_res: " + mv_origin_res) + assertTrue((mv_origin_res == [] && origin_res == []) || (mv_origin_res.size() == origin_res.size())) + for (int row = 0; row < mv_origin_res.size(); row++) { + assertTrue(mv_origin_res[row].size() == origin_res[row].size()) + for (int col = 0; col < mv_origin_res[row].size(); col++) { + assertTrue(mv_origin_res[row][col] == origin_res[row][col]) + } + } + } + + + // create base first level mv + create_async_mv(db, "join_mv1", """ + SELECT l_orderkey, l_linenumber, l_partkey, o_orderkey, o_custkey + FROM lineitem_2 INNER JOIN orders_2 + ON l_orderkey = o_orderkey; + """) + + // create second level mv based on first level mv + create_async_mv(db, "agg_mv2", """ + SELECT + l_orderkey, + l_linenumber, + o_orderkey, + sum(l_partkey) AS total_revenue, + max(o_custkey) AS max_discount + FROM join_mv1 + GROUP BY l_orderkey, l_linenumber, o_orderkey; + """) + + // create third level mv based on second level mv + create_async_mv(db, "join_agg_mv3", """ + SELECT + l_orderkey, + sum(total_revenue) AS total_revenue, + max(max_discount) AS max_discount + FROM agg_mv2 + GROUP BY l_orderkey; + """) + + def query = """ + SELECT l_orderkey, sum(l_partkey) AS total_revenue, max(o_custkey) AS max_discount FROM lineitem_2 INNER JOIN orders_2 ON l_orderkey = o_orderkey GROUP BY l_orderkey + """ + + sql """set enable_materialized_view_nest_rewrite = false;""" + // Just first level mv rewrite successfully, second and third level mv should rewriten fail + mv_rewrite_fail(query, "agg_mv2") + mv_rewrite_fail(query, "join_agg_mv3") + mv_rewrite_success(query, "join_mv1") + compare_res(query + " order by 1,2,3") + + + sql """set enable_materialized_view_nest_rewrite = true;""" + // All mv rewrite successfully but only thirst level mv can be chosen by cbo + mv_rewrite_success_without_check_chosen(query, "join_mv1") + mv_rewrite_success_without_check_chosen(query, "agg_mv2") + mv_rewrite_success(query, "join_agg_mv3") + compare_res(query + " order by 1,2,3") + + +} --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org