This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5cb5241a9ea [feature](mtmv) materialized view rewrite framework
(#27059)
5cb5241a9ea is described below
commit 5cb5241a9ea9cc2bdea5c04ad451ec113e4e2248
Author: seawinde <[email protected]>
AuthorDate: Mon Nov 27 11:15:54 2023 +0800
[feature](mtmv) materialized view rewrite framework (#27059)
materialized view rewrite framework, support to query rewrite by struct
info.
The idea is from "Optimizing Queries Using Materialized Views- A Practical,
Scalable Solution"
---
.../org/apache/doris/nereids/CascadesContext.java | 12 +
.../org/apache/doris/nereids/NereidsPlanner.java | 4 +
.../jobs/cascades/OptimizeGroupExpressionJob.java | 3 +
.../java/org/apache/doris/nereids/memo/Group.java | 11 +
.../org/apache/doris/nereids/rules/RuleSet.java | 9 +
.../org/apache/doris/nereids/rules/RuleType.java | 17 ++
.../mv/AbstractMaterializedViewAggregateRule.java | 25 ++
.../mv/AbstractMaterializedViewJoinRule.java | 62 +++++
.../mv/AbstractMaterializedViewRule.java | 253 +++++++++++++++++++++
.../rules/exploration/mv/EquivalenceClass.java | 92 ++++++++
.../nereids/rules/exploration/mv/Mapping.java | 144 ++++++++++++
.../exploration/mv/MaterializationContext.java | 68 ++++++
.../mv/MaterializedViewAggregateRule.java | 33 +++
.../mv/MaterializedViewProjectJoinRule.java | 47 ++++
.../exploration/mv/MaterializedViewScanRule.java | 34 +++
.../nereids/rules/exploration/mv/Predicates.java | 131 +++++++++++
.../rules/exploration/mv/RelationMapping.java | 63 +++++
.../nereids/rules/exploration/mv/SlotMapping.java | 49 ++++
.../nereids/rules/exploration/mv/StructInfo.java | 106 +++++++++
.../expressions/visitor/ExpressionVisitors.java | 49 ++++
.../apache/doris/nereids/util/ExpressionUtils.java | 28 +++
.../java/org/apache/doris/qe/SessionVariable.java | 25 ++
22 files changed, 1265 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index ebef71feb85..3c21a988fb5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -44,6 +44,7 @@ import org.apache.doris.nereids.properties.PhysicalProperties;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleSet;
import
org.apache.doris.nereids.rules.analysis.BindRelation.CustomTableResolver;
+import org.apache.doris.nereids.rules.exploration.mv.MaterializationContext;
import org.apache.doris.nereids.trees.expressions.CTEId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.NamedExpression;
@@ -112,6 +113,8 @@ public class CascadesContext implements ScheduleContext {
private final Optional<CTEId> currentTree;
private final Optional<CascadesContext> parent;
+ private final List<MaterializationContext> materializationContexts;
+
/**
* Constructor of OptimizerContext.
*
@@ -133,6 +136,7 @@ public class CascadesContext implements ScheduleContext {
this.currentJobContext = new JobContext(this, requireProperties,
Double.MAX_VALUE);
this.subqueryExprIsAnalyzed = new HashMap<>();
this.runtimeFilterContext = new
RuntimeFilterContext(getConnectContext().getSessionVariable());
+ this.materializationContexts = new ArrayList<>();
}
/**
@@ -309,6 +313,14 @@ public class CascadesContext implements ScheduleContext {
this.outerScope = Optional.ofNullable(outerScope);
}
+ public List<MaterializationContext> getMaterializationContexts() {
+ return materializationContexts;
+ }
+
+ public void addMaterializationContext(MaterializationContext
materializationContext) {
+ this.materializationContexts.add(materializationContext);
+ }
+
/**
* getAndCacheSessionVariable
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index bf870f9dd6b..63aadc7af19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -259,6 +259,10 @@ public class NereidsPlanner extends Planner {
if (statementContext.getConnectContext().getTables() != null) {
cascadesContext.setTables(statementContext.getConnectContext().getTables());
}
+ if
(statementContext.getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite())
{
+ // TODO Pre handle materialized view to materializationContext and
+ // call cascadesContext.addMaterializationContext() to add it
+ }
}
private void analyze() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
index 38c0c4484bc..08b7731fd09 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/cascades/OptimizeGroupExpressionJob.java
@@ -46,6 +46,9 @@ public class OptimizeGroupExpressionJob extends Job {
countJobExecutionTimesOfGroupExpressions(groupExpression);
List<Rule> implementationRules = getRuleSet().getImplementationRules();
List<Rule> explorationRules = getExplorationRules();
+ if
(context.getCascadesContext().getConnectContext().getSessionVariable().isEnableMaterializedViewRewrite())
{
+ explorationRules.addAll(getRuleSet().getMaterializedViewRules());
+ }
for (Rule rule : explorationRules) {
if (rule.isInvalid(disableRules, groupExpression)) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
index ece30e1f4da..8d09c8afac2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/memo/Group.java
@@ -22,6 +22,7 @@ import org.apache.doris.nereids.cost.Cost;
import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.properties.PhysicalProperties;
+import org.apache.doris.nereids.rules.exploration.mv.StructInfo;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.JoinType;
import org.apache.doris.nereids.trees.plans.Plan;
@@ -76,6 +77,8 @@ public class Group {
private int chosenGroupExpressionId = -1;
+ private Optional<StructInfo> structInfo = Optional.empty();
+
/**
* Constructor for Group.
*
@@ -538,4 +541,12 @@ public class Group {
return TreeStringUtils.treeString(this, toString, getChildren,
getExtraPlans, displayExtraPlan);
}
+
+ public Optional<StructInfo> getStructInfo() {
+ return structInfo;
+ }
+
+ public void setStructInfo(StructInfo structInfo) {
+ this.structInfo = Optional.ofNullable(structInfo);
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
index 736ad7f7a87..ed2bc775e25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleSet.java
@@ -40,6 +40,7 @@ import
org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughInn
import
org.apache.doris.nereids.rules.exploration.join.PushDownProjectThroughSemiJoin;
import
org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTranspose;
import
org.apache.doris.nereids.rules.exploration.join.SemiJoinSemiJoinTransposeProject;
+import
org.apache.doris.nereids.rules.exploration.mv.MaterializedViewProjectJoinRule;
import org.apache.doris.nereids.rules.implementation.AggregateStrategies;
import
org.apache.doris.nereids.rules.implementation.LogicalAssertNumRowsToPhysicalAssertNumRows;
import
org.apache.doris.nereids.rules.implementation.LogicalCTEAnchorToPhysicalCTEAnchor;
@@ -220,6 +221,10 @@ public class RuleSet {
.add(JoinCommute.BUSHY.build())
.build();
+ public static final List<Rule> MATERIALIZED_VIEW_RULES =
planRuleFactories()
+ .add(MaterializedViewProjectJoinRule.INSTANCE)
+ .build();
+
public List<Rule> getDPHypReorderRules() {
return DPHYP_REORDER_RULES;
}
@@ -240,6 +245,10 @@ public class RuleSet {
return IMPLEMENTATION_RULES;
}
+ public List<Rule> getMaterializedViewRules() {
+ return MATERIALIZED_VIEW_RULES;
+ }
+
public static RuleFactories planRuleFactories() {
return new RuleFactories();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 40a63a056c0..e91b91360e5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -227,6 +227,7 @@ public enum RuleType {
MATERIALIZED_INDEX_PROJECT_SCAN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_PROJECT_FILTER_SCAN(RuleTypeClass.REWRITE),
MATERIALIZED_INDEX_FILTER_PROJECT_SCAN(RuleTypeClass.REWRITE),
+
OLAP_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
FILE_SCAN_PARTITION_PRUNE(RuleTypeClass.REWRITE),
PUSH_CONJUNCTS_INTO_JDBC_SCAN(RuleTypeClass.REWRITE),
@@ -321,6 +322,22 @@ public enum RuleType {
EAGER_SPLIT(RuleTypeClass.EXPLORATION),
EXPLORATION_SENTINEL(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_PROJECT_JOIN(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_FILTER_JOIN(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_PROJECT_FILTER_JOIN(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_FILTER_PROJECT_JOIN(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_ONLY_JOIN(RuleTypeClass.EXPLORATION),
+
+ MATERIALIZED_VIEW_PROJECT_AGGREGATE(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_FILTER_AGGREGATE(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_PROJECT_FILTER_AGGREGATE(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_FILTER_PROJECT_AGGREGATE(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_ONLY_AGGREGATE(RuleTypeClass.EXPLORATION),
+
+ MATERIALIZED_VIEW_FILTER_SCAN(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_PROJECT_SCAN(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_FILTER_PROJECT_SCAN(RuleTypeClass.EXPLORATION),
+ MATERIALIZED_VIEW_PROJECT_FILTER_SCAN(RuleTypeClass.EXPLORATION),
// implementation rules
LOGICAL_ONE_ROW_RELATION_TO_PHYSICAL_ONE_ROW_RELATION(RuleTypeClass.IMPLEMENTATION),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
new file mode 100644
index 00000000000..a9a8b754d33
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+/**
+ * AbstractMaterializedViewAggregateRule
+ * This is responsible for common aggregate rewriting
+ * */
+public abstract class AbstractMaterializedViewAggregateRule extends
AbstractMaterializedViewRule {
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
new file mode 100644
index 00000000000..17df9ef88ed
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewJoinRule.java
@@ -0,0 +1,62 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import java.util.List;
+
+/**
+ * AbstractMaterializedViewJoinRule
+ * This is responsible for common join rewriting
+ */
+public abstract class AbstractMaterializedViewJoinRule extends
AbstractMaterializedViewRule {
+
+ @Override
+ protected Plan rewriteQueryByView(MatchMode matchMode,
+ StructInfo queryStructInfo,
+ StructInfo viewStructInfo,
+ RelationMapping queryToViewTableMappings,
+ Plan tempRewritedPlan) {
+
+ // Rewrite top projects, represent the query projects by view
+ List<NamedExpression> expressions = rewriteExpression(
+ queryStructInfo.getExpressions(),
+ queryStructInfo,
+ viewStructInfo,
+ queryToViewTableMappings,
+ tempRewritedPlan
+ );
+ // Can not rewrite, bail out
+ if (expressions == null) {
+ return null;
+ }
+ return new LogicalProject<>(expressions, tempRewritedPlan);
+ }
+
+ // Check join is whether valid or not. Support join's input can not
contain aggregate
+ // Only support project, filter, join, logical relation node and
+ // join condition should be slot reference equals currently
+ @Override
+ protected boolean checkPattern(StructInfo structInfo) {
+ // TODO Should get struct info from hyper graph and check
+ return false;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
new file mode 100644
index 00000000000..c52acdb3fda
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewRule.java
@@ -0,0 +1,253 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.memo.Group;
+import
org.apache.doris.nereids.rules.exploration.mv.Mapping.ExpressionIndexMapping;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.NamedExpression;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Sets;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.stream.Collectors;
+
+/**
+ * The abstract class for all materialized view rules
+ */
+public abstract class AbstractMaterializedViewRule {
+
+ /**
+ * The abstract template method for query rewrite, it contains the main
logic and different query
+ * pattern should override the sub logic.
+ */
+ protected List<Plan> rewrite(Plan queryPlan, CascadesContext
cascadesContext) {
+ List<MaterializationContext> materializationContexts =
cascadesContext.getMaterializationContexts();
+ List<Plan> rewriteResults = new ArrayList<>();
+ if (materializationContexts.isEmpty()) {
+ return rewriteResults;
+ }
+ StructInfo queryStructInfo = extractStructInfo(queryPlan,
cascadesContext);
+ // Check query queryPlan
+ if (!checkPattern(queryStructInfo)) {
+ return rewriteResults;
+ }
+
+ for (MaterializationContext materializationContext :
materializationContexts) {
+ Plan mvPlan = materializationContext.getMvPlan();
+ StructInfo viewStructInfo = extractStructInfo(mvPlan,
cascadesContext);
+ if (!checkPattern(viewStructInfo)) {
+ continue;
+ }
+ if
(!StructInfo.isGraphLogicalEquals(queryStructInfo.getHyperGraph(),
viewStructInfo.getHyperGraph())) {
+ continue;
+ }
+ MatchMode matchMode =
decideMatchMode(queryStructInfo.getRelations(), viewStructInfo.getRelations());
+ if (MatchMode.NOT_MATCH == matchMode) {
+ continue;
+ }
+ List<RelationMapping> queryToViewTableMappings =
+ RelationMapping.generate(queryStructInfo.getRelations(),
viewStructInfo.getRelations());
+ for (RelationMapping queryToViewTableMapping :
queryToViewTableMappings) {
+ SplitPredicate compensatePredicates =
predicatesCompensate(queryStructInfo, viewStructInfo,
+ queryToViewTableMapping);
+ // Can not compensate, bail out
+ if (compensatePredicates == null ||
compensatePredicates.isEmpty()) {
+ continue;
+ }
+ Plan rewritedPlan;
+ Plan mvScan = materializationContext.getScanPlan();
+ if (compensatePredicates.isAlwaysTrue()) {
+ rewritedPlan = mvScan;
+ } else {
+ // Try to rewrite compensate predicates by using mv scan
+ List<NamedExpression> rewriteCompensatePredicates =
rewriteExpression(
+ compensatePredicates.toList(),
+ queryStructInfo,
+ viewStructInfo,
+ queryToViewTableMapping,
+ mvScan);
+ if (rewriteCompensatePredicates.isEmpty()) {
+ continue;
+ }
+ rewritedPlan = new
LogicalFilter<>(Sets.newHashSet(rewriteCompensatePredicates), mvScan);
+ }
+ // Rewrite query by view
+ rewritedPlan = rewriteQueryByView(matchMode, queryStructInfo,
viewStructInfo,
+ queryToViewTableMapping, rewritedPlan);
+ if (rewritedPlan == null) {
+ continue;
+ }
+ rewriteResults.add(rewritedPlan);
+ }
+ }
+ return rewriteResults;
+ }
+
+ /**Rewrite query by view, for aggregate or join rewriting should be
different inherit class implementation*/
+ protected Plan rewriteQueryByView(MatchMode matchMode,
+ StructInfo queryStructInfo,
+ StructInfo viewStructInfo,
+ RelationMapping queryToViewTableMappings,
+ Plan tempRewritedPlan) {
+ return tempRewritedPlan;
+ }
+
+ /**Use target output expression to represent the source expression*/
+ protected List<NamedExpression> rewriteExpression(List<? extends
Expression> sourceExpressions,
+ StructInfo sourceStructInfo,
+ StructInfo targetStructInfo,
+ RelationMapping sourceToTargetMapping,
+ Plan targetScanNode) {
+ // TODO represent the sourceExpressions by using target scan node
+ // Firstly, rewrite the target plan output expression using query with
inverse mapping
+ // then try to use the mv expression to represent the query. if any of
source expressions
+ // can not be represented by mv, return null
+ //
+ // example as following:
+ // source target
+ // project(slot 1, 2) project(slot 3, 2, 1)
+ // scan(table) scan(table)
+ //
+ // transform source to:
+ // project(slot 2, 1)
+ // target
+ List<? extends Expression> targetTopExpressions =
targetStructInfo.getExpressions();
+ List<? extends Expression> shuttledTargetExpressions =
ExpressionUtils.shuttleExpressionWithLineage(
+ targetTopExpressions, targetStructInfo.getOriginalPlan(),
Sets.newHashSet(), Sets.newHashSet());
+ SlotMapping sourceToTargetSlotMapping =
SlotMapping.generate(sourceToTargetMapping);
+ // mv sql plan expressions transform to query based
+ List<? extends Expression> queryBasedExpressions =
ExpressionUtils.permute(shuttledTargetExpressions,
+ sourceToTargetSlotMapping.inverse());
+ // mv sql query based expression and index mapping
+ ExpressionIndexMapping.generate(queryBasedExpressions);
+ // TODO visit source expression and replace the expression with
expressionIndexMapping
+ return ImmutableList.of();
+ }
+
+ /**
+ * Compensate mv predicates by query predicates, compensate predicate
result is query based.
+ * Such as a > 5 in mv, and a > 10 in query, the compensatory predicate is
a > 10.
+ * For another example as following:
+ * predicate a = b in mv, and a = b and c = d in query, the compensatory
predicate is c = d
+ */
+ protected SplitPredicate predicatesCompensate(
+ StructInfo queryStructInfo,
+ StructInfo viewStructInfo,
+ RelationMapping queryToViewTableMapping
+ ) {
+ // TODO Equal predicate compensate
+ EquivalenceClass queryEquivalenceClass =
queryStructInfo.getEquivalenceClass();
+ EquivalenceClass viewEquivalenceClass =
viewStructInfo.getEquivalenceClass();
+ if (queryEquivalenceClass.isEmpty()
+ && !viewEquivalenceClass.isEmpty()) {
+ return null;
+ }
+ // TODO range predicates and residual predicates compensate
+ return SplitPredicate.empty();
+ }
+
+ /**
+ * Decide the match mode
+ * @see MatchMode
+ */
+ private MatchMode decideMatchMode(List<CatalogRelation> queryRelations,
List<CatalogRelation> viewRelations) {
+ List<TableIf> queryTableRefs = queryRelations
+ .stream()
+ .map(CatalogRelation::getTable)
+ .collect(Collectors.toList());
+ List<TableIf> viewTableRefs = viewRelations
+ .stream()
+ .map(CatalogRelation::getTable)
+ .collect(Collectors.toList());
+ boolean sizeSame = viewTableRefs.size() == queryTableRefs.size();
+ boolean queryPartial = viewTableRefs.containsAll(queryTableRefs);
+ if (!sizeSame && queryPartial) {
+ return MatchMode.QUERY_PARTIAL;
+ }
+ boolean viewPartial = queryTableRefs.containsAll(viewTableRefs);
+ if (!sizeSame && viewPartial) {
+ return MatchMode.VIEW_PARTIAL;
+ }
+ if (sizeSame && queryPartial && viewPartial) {
+ return MatchMode.COMPLETE;
+ }
+ return MatchMode.NOT_MATCH;
+ }
+
+ /**
+ * Extract struct info from plan, support to get struct info from logical
plan or plan in group.
+ */
+ protected StructInfo extractStructInfo(Plan plan, CascadesContext
cascadesContext) {
+
+ if (plan.getGroupExpression().isPresent()
+ &&
plan.getGroupExpression().get().getOwnerGroup().getStructInfo().isPresent()) {
+ Group belongGroup =
plan.getGroupExpression().get().getOwnerGroup();
+ return belongGroup.getStructInfo().get();
+ } else {
+ // TODO build graph from plan and extract struct from graph and
set to group if exist
+ // Should get structInfo from hyper graph and add into current
group
+ StructInfo structInfo = StructInfo.of(plan);
+ if (plan.getGroupExpression().isPresent()) {
+
plan.getGroupExpression().get().getOwnerGroup().setStructInfo(structInfo);
+ }
+ return structInfo;
+ }
+ }
+
+ /**
+ * Check the pattern of query or materializedView is supported or not.
+ */
+ protected boolean checkPattern(StructInfo structInfo) {
+ if (structInfo.getRelations().isEmpty()) {
+ return false;
+ }
+ return false;
+ }
+
+ /**
+ * Query and mv match node
+ */
+ protected enum MatchMode {
+ /**
+ * The tables in query are same to the tables in view
+ */
+ COMPLETE,
+ /**
+ * The tables in query contains all the tables in view
+ */
+ VIEW_PARTIAL,
+ /**
+ * The tables in view contains all the tables in query
+ */
+ QUERY_PARTIAL,
+ /**
+ * Except for COMPLETE and VIEW_PARTIAL and QUERY_PARTIAL
+ */
+ NOT_MATCH
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java
new file mode 100644
index 00000000000..d140582aa64
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java
@@ -0,0 +1,92 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * EquivalenceClass, this is used for equality propagation when predicate
compensation
+ */
+public class EquivalenceClass {
+
+ private final Map<SlotReference, Set<SlotReference>> equivalenceSlotMap =
new LinkedHashMap<>();
+
+ public EquivalenceClass() {
+ }
+
+ /**
+ * EquivalenceClass
+ */
+ public void addEquivalenceClass(SlotReference leftSlot, SlotReference
rightSlot) {
+
+ Set<SlotReference> leftSlotSet = equivalenceSlotMap.get(leftSlot);
+ Set<SlotReference> rightSlotSet = equivalenceSlotMap.get(rightSlot);
+ if (leftSlotSet != null && rightSlotSet != null) {
+ // Both present, we need to merge
+ if (leftSlotSet.size() < rightSlotSet.size()) {
+ // We swap them to merge
+ Set<SlotReference> tmp = rightSlotSet;
+ rightSlotSet = leftSlotSet;
+ leftSlotSet = tmp;
+ }
+ for (SlotReference newRef : rightSlotSet) {
+ leftSlotSet.add(newRef);
+ equivalenceSlotMap.put(newRef, leftSlotSet);
+ }
+ } else if (leftSlotSet != null) {
+ // leftSlotSet present, we need to merge into it
+ leftSlotSet.add(rightSlot);
+ equivalenceSlotMap.put(rightSlot, leftSlotSet);
+ } else if (rightSlotSet != null) {
+ // rightSlotSet present, we need to merge into it
+ rightSlotSet.add(leftSlot);
+ equivalenceSlotMap.put(leftSlot, rightSlotSet);
+ } else {
+ // None are present, add to same equivalence class
+ Set<SlotReference> equivalenceClass = new LinkedHashSet<>();
+ equivalenceClass.add(leftSlot);
+ equivalenceClass.add(rightSlot);
+ equivalenceSlotMap.put(leftSlot, equivalenceClass);
+ equivalenceSlotMap.put(rightSlot, equivalenceClass);
+ }
+ }
+
+ public Map<SlotReference, Set<SlotReference>> getEquivalenceSlotMap() {
+ return equivalenceSlotMap;
+ }
+
+ public boolean isEmpty() {
+ return equivalenceSlotMap.isEmpty();
+ }
+
+ /**
+ * EquivalenceClass
+ */
+ public List<Set<SlotReference>> getEquivalenceValues() {
+ List<Set<SlotReference>> values = new ArrayList<>();
+ equivalenceSlotMap.values().forEach(each -> values.add(each));
+ return values;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java
new file mode 100644
index 00000000000..487fc92ce50
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Mapping.java
@@ -0,0 +1,144 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.trees.expressions.ExprId;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.plans.RelationId;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.Multimap;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * Mapping slot from query to view or inversely,
+ * it can also represent the mapping from slot to it's index
+ */
+public abstract class Mapping {
+
+ /**
+ * The relation for mapping
+ */
+ public static final class MappedRelation {
+ public final RelationId relationId;
+ public final CatalogRelation belongedRelation;
+
+ public MappedRelation(RelationId relationId, CatalogRelation
belongedRelation) {
+ this.relationId = relationId;
+ this.belongedRelation = belongedRelation;
+ }
+
+ public MappedRelation of(RelationId relationId, CatalogRelation
belongedRelation) {
+ return new MappedRelation(relationId, belongedRelation);
+ }
+
+ public RelationId getRelationId() {
+ return relationId;
+ }
+
+ public CatalogRelation getBelongedRelation() {
+ return belongedRelation;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MappedRelation that = (MappedRelation) o;
+ return Objects.equals(relationId, that.relationId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(relationId);
+ }
+ }
+
+ /**
+ * The slot for mapping
+ */
+ public static final class MappedSlot {
+
+ public final ExprId exprId;
+ public final CatalogRelation belongedRelation;
+
+ public MappedSlot(ExprId exprId, CatalogRelation belongedRelation) {
+ this.exprId = exprId;
+ this.belongedRelation = belongedRelation;
+ }
+
+ public MappedSlot of(ExprId exprId, CatalogRelation belongedRelation) {
+ return new MappedSlot(exprId, belongedRelation);
+ }
+
+ public ExprId getExprId() {
+ return exprId;
+ }
+
+ public CatalogRelation getBelongedRelation() {
+ return belongedRelation;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ MappedSlot that = (MappedSlot) o;
+ return Objects.equals(exprId, that.exprId);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(exprId);
+ }
+ }
+
+ /**
+ * Expression and it's index mapping
+ */
+ public static class ExpressionIndexMapping extends Mapping {
+ private final Multimap<Expression, Integer> expressionIndexMapping;
+
+ public ExpressionIndexMapping(Multimap<Expression, Integer>
expressionIndexMapping) {
+ this.expressionIndexMapping = expressionIndexMapping;
+ }
+
+ public Multimap<Expression, Integer> getExpressionIndexMapping() {
+ return expressionIndexMapping;
+ }
+
+ public static ExpressionIndexMapping generate(List<? extends
Expression> expressions) {
+ Multimap<Expression, Integer> expressionIndexMapping =
ArrayListMultimap.create();
+ for (int i = 0; i < expressions.size(); i++) {
+ expressionIndexMapping.put(expressions.get(i), i);
+ }
+ return new ExpressionIndexMapping(expressionIndexMapping);
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
new file mode 100644
index 00000000000..b0de1ccfa46
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializationContext.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.catalog.Table;
+import org.apache.doris.catalog.View;
+import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.memo.GroupId;
+import org.apache.doris.nereids.trees.plans.Plan;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Maintain the context for query rewrite by materialized view
+ */
+public class MaterializationContext {
+
+ // TODO add MaterializedView class
+ private final Plan mvPlan;
+ private final CascadesContext context;
+ private final List<Table> baseTables;
+ private final List<View> baseViews;
+ // Group ids that are rewritten by this mv to reduce rewrite times
+ private final Set<GroupId> matchedGroups = new HashSet<>();
+ private final Plan scanPlan;
+
+ public MaterializationContext(Plan mvPlan, CascadesContext context,
+ List<Table> baseTables, List<View> baseViews, Plan scanPlan) {
+ this.mvPlan = mvPlan;
+ this.context = context;
+ this.baseTables = baseTables;
+ this.baseViews = baseViews;
+ this.scanPlan = scanPlan;
+ }
+
+ public Set<GroupId> getMatchedGroups() {
+ return matchedGroups;
+ }
+
+ public void addMatchedGroup(GroupId groupId) {
+ matchedGroups.add(groupId);
+ }
+
+ public Plan getMvPlan() {
+ return mvPlan;
+ }
+
+ public Plan getScanPlan() {
+ return scanPlan;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
new file mode 100644
index 00000000000..ce9c208e5f5
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewAggregateRule.java
@@ -0,0 +1,33 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
+
+import java.util.List;
+
+/**
+ * This is responsible for aggregate rewriting according to different pattern
+ * */
+public class MaterializedViewAggregateRule extends
AbstractMaterializedViewAggregateRule implements RewriteRuleFactory {
+ @Override
+ public List<Rule> buildRules() {
+ return null;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
new file mode 100644
index 00000000000..92f102dc1de
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectJoinRule.java
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.RulePromise;
+import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+
+/**
+ * This is responsible for join rewriting according to different pattern
+ * */
+public class MaterializedViewProjectJoinRule extends
AbstractMaterializedViewJoinRule implements RewriteRuleFactory {
+
+ public static final MaterializedViewProjectJoinRule INSTANCE = new
MaterializedViewProjectJoinRule();
+
+ @Override
+ public List<Rule> buildRules() {
+ return ImmutableList.of(
+ logicalProject(logicalJoin(any(), any())).thenApplyMulti(ctx
-> {
+ LogicalProject<LogicalJoin<Plan, Plan>> root = ctx.root;
+ return rewrite(root, ctx.cascadesContext);
+ }).toRule(RuleType.MATERIALIZED_VIEW_ONLY_JOIN,
RulePromise.EXPLORE));
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
new file mode 100644
index 00000000000..c5909822adb
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewScanRule.java
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.rules.Rule;
+import org.apache.doris.nereids.rules.rewrite.RewriteRuleFactory;
+
+import java.util.List;
+
+/**
+ * This is responsible for single table rewriting according to different
pattern
+ * */
+public class MaterializedViewScanRule extends AbstractMaterializedViewRule
implements RewriteRuleFactory {
+
+ @Override
+ public List<Rule> buildRules() {
+ return null;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
new file mode 100644
index 00000000000..40b91994be7
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
+import
org.apache.doris.nereids.trees.expressions.visitor.ExpressionVisitors.PredicatesSpliter;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import com.google.common.collect.ImmutableList;
+
+import java.util.List;
+import java.util.Set;
+
+/**
+ * This record the predicates which can be pulled up or some other type
predicates
+ * */
+public class Predicates {
+
+ // Predicates that can be pulled up
+ private final Set<Expression> pulledUpPredicates;
+
+ public Predicates(Set<Expression> pulledUpPredicates) {
+ this.pulledUpPredicates = pulledUpPredicates;
+ }
+
+ public static Predicates of(Set<Expression> pulledUpPredicates) {
+ return new Predicates(pulledUpPredicates);
+ }
+
+ public Set<Expression> getPulledUpPredicates() {
+ return pulledUpPredicates;
+ }
+
+ public Expression composedExpression() {
+ return ExpressionUtils.and(pulledUpPredicates);
+ }
+
+ /**
+ * Split the expression to equal, range and residual predicate.
+ * */
+ public static SplitPredicate splitPredicates(Expression expression) {
+ PredicatesSpliter predicatesSplit = new PredicatesSpliter(expression);
+ expression.accept(predicatesSplit, null);
+ return predicatesSplit.getSplitPredicate();
+ }
+
+ /**
+ * The split different representation for predicate expression, such as
equal, range and residual predicate.
+ * */
+ public static final class SplitPredicate {
+ private final Expression equalPredicates;
+ private final Expression rangePredicates;
+ private final Expression residualPredicates;
+
+ public SplitPredicate(Expression equalPredicates, Expression
rangePredicates, Expression residualPredicates) {
+ this.equalPredicates = equalPredicates;
+ this.rangePredicates = rangePredicates;
+ this.residualPredicates = residualPredicates;
+ }
+
+ public Expression getEqualPredicates() {
+ return equalPredicates;
+ }
+
+ public Expression getRangePredicates() {
+ return rangePredicates;
+ }
+
+ public Expression getResidualPredicates() {
+ return residualPredicates;
+ }
+
+ public static SplitPredicate empty() {
+ return new SplitPredicate(null, null, null);
+ }
+
+ /**
+ * SplitPredicate construct
+ * */
+ public static SplitPredicate of(Expression equalPredicates,
+ Expression rangePredicates,
+ Expression residualPredicates) {
+ return new SplitPredicate(equalPredicates, rangePredicates,
residualPredicates);
+ }
+
+ /**
+ * isEmpty
+ * */
+ public boolean isEmpty() {
+ return equalPredicates == null
+ && rangePredicates == null
+ && residualPredicates == null;
+ }
+
+ public Expression composedExpression() {
+ return ExpressionUtils.and(equalPredicates, rangePredicates,
residualPredicates);
+ }
+
+ public List<Expression> toList() {
+ return ImmutableList.of(equalPredicates, rangePredicates,
residualPredicates);
+ }
+
+ /**
+ * Check the predicates in SplitPredicate is whether all true or not
+ */
+ public boolean isAlwaysTrue() {
+ return equalPredicates instanceof BooleanLiteral
+ && rangePredicates instanceof BooleanLiteral
+ && residualPredicates instanceof BooleanLiteral
+ && ((BooleanLiteral) equalPredicates).getValue()
+ && ((BooleanLiteral) rangePredicates).getValue()
+ && ((BooleanLiteral) residualPredicates).getValue();
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java
new file mode 100644
index 00000000000..ea91104a246
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/RelationMapping.java
@@ -0,0 +1,63 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+
+import com.google.common.collect.ArrayListMultimap;
+import com.google.common.collect.BiMap;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Multimap;
+
+import java.util.List;
+
+/**
+ * Relation mapping
+ * such as query pattern is a1 left join a2 left join b
+ * view pattern is a1 left join a2 left join b. the mapping will be
+ * [{a1:a1, a2:a2, b:b}, {a1:a2, a2:a1, b:b}]
+ */
+public class RelationMapping extends Mapping {
+
+ private final BiMap<MappedRelation, MappedRelation> mappedRelationMap;
+
+ public RelationMapping(BiMap<MappedRelation, MappedRelation>
mappedRelationMap) {
+ this.mappedRelationMap = mappedRelationMap;
+ }
+
+ public BiMap<MappedRelation, MappedRelation> getMappedRelationMap() {
+ return mappedRelationMap;
+ }
+
+ /**
+ * Generate mapping according to source and target relation
+ */
+ public static List<RelationMapping> generate(List<CatalogRelation> source,
List<CatalogRelation> target) {
+ Multimap<TableIf, CatalogRelation> queryTableRelationIdMap =
ArrayListMultimap.create();
+ for (CatalogRelation relation : source) {
+ queryTableRelationIdMap.put(relation.getTable(), relation);
+ }
+ Multimap<TableIf, CatalogRelation> viewTableRelationIdMap =
ArrayListMultimap.create();
+ for (CatalogRelation relation : target) {
+ viewTableRelationIdMap.put(relation.getTable(), relation);
+ }
+ // todo generate relation map
+ return ImmutableList.of();
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java
new file mode 100644
index 00000000000..7c50d79c6a2
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/SlotMapping.java
@@ -0,0 +1,49 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import com.google.common.collect.BiMap;
+
+/**
+ * SlotMapping, this is open generated from relationMapping
+ */
+public class SlotMapping extends Mapping {
+
+ private final BiMap<MappedSlot, MappedSlot> relationSlotMap;
+
+ public SlotMapping(BiMap<MappedSlot, MappedSlot> relationSlotMap) {
+ this.relationSlotMap = relationSlotMap;
+ }
+
+ public BiMap<MappedSlot, MappedSlot> getRelationSlotMap() {
+ return relationSlotMap;
+ }
+
+ public SlotMapping inverse() {
+ return SlotMapping.of(relationSlotMap.inverse());
+ }
+
+ public static SlotMapping of(BiMap<MappedSlot, MappedSlot>
relationSlotMap) {
+ return new SlotMapping(relationSlotMap);
+ }
+
+ public static SlotMapping generate(RelationMapping relationMapping) {
+ // TODO implement
+ return SlotMapping.of(null);
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
new file mode 100644
index 00000000000..141b8a98bcc
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/StructInfo.java
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.nereids.rules.exploration.mv;
+
+import org.apache.doris.nereids.jobs.joinorder.hypergraph.HyperGraph;
+import org.apache.doris.nereids.memo.Group;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates.SplitPredicate;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.Expression;
+import org.apache.doris.nereids.trees.expressions.SlotReference;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
+import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import java.util.List;
+
+/**
+ * StructInfo
+ */
+public class StructInfo {
+ private final List<CatalogRelation> relations;
+ private final Predicates predicates;
+ // Used by predicate compensation
+ private final EquivalenceClass equivalenceClass;
+ private final Plan originalPlan;
+ private final HyperGraph hyperGraph;
+
+ private StructInfo(List<CatalogRelation> relations,
+ Predicates predicates,
+ Plan originalPlan,
+ HyperGraph hyperGraph) {
+ this.relations = relations;
+ this.predicates = predicates;
+ this.originalPlan = originalPlan;
+ this.hyperGraph = hyperGraph;
+ // construct equivalenceClass according to equals predicates
+ this.equivalenceClass = new EquivalenceClass();
+ SplitPredicate splitPredicate =
Predicates.splitPredicates(predicates.composedExpression());
+ for (Expression expression :
ExpressionUtils.extractConjunction(splitPredicate.getEqualPredicates())) {
+ EqualTo equalTo = (EqualTo) expression;
+ equivalenceClass.addEquivalenceClass(
+ (SlotReference) equalTo.getArguments().get(0),
+ (SlotReference) equalTo.getArguments().get(1));
+ }
+ }
+
+ public static StructInfo of(Plan originalPlan) {
+ // TODO build graph from original plan and get relations and
predicates from graph
+ return new StructInfo(null, null, originalPlan, null);
+ }
+
+ public static StructInfo of(Group group) {
+ // TODO build graph from original plan and get relations and
predicates from graph
+ return new StructInfo(null, null,
group.getLogicalExpression().getPlan(), null);
+ }
+
+ public List<CatalogRelation> getRelations() {
+ return relations;
+ }
+
+ public Predicates getPredicates() {
+ return predicates;
+ }
+
+ public EquivalenceClass getEquivalenceClass() {
+ return equivalenceClass;
+ }
+
+ public Plan getOriginalPlan() {
+ return originalPlan;
+ }
+
+ public HyperGraph getHyperGraph() {
+ return hyperGraph;
+ }
+
+ public List<? extends Expression> getExpressions() {
+ return originalPlan instanceof LogicalProject
+ ? ((LogicalProject<Plan>) originalPlan).getProjects() :
originalPlan.getOutput();
+ }
+
+ /**
+ * Judge the source graph logical is whether the same as target
+ * For inner join should judge only the join tables,
+ * for other join type should also judge the join direction, it's input
filter that can not be pulled up etc.
+ * */
+ public static boolean isGraphLogicalEquals(HyperGraph source, HyperGraph
target) {
+ return false;
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
index 513da0e93d9..86949f63e16 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/visitor/ExpressionVisitors.java
@@ -17,9 +17,16 @@
package org.apache.doris.nereids.trees.expressions.visitor;
+import org.apache.doris.nereids.rules.exploration.mv.Predicates;
+import org.apache.doris.nereids.trees.expressions.ComparisonPredicate;
+import org.apache.doris.nereids.trees.expressions.EqualTo;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.WindowExpression;
import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
+import org.apache.doris.nereids.util.ExpressionUtils;
+
+import java.util.ArrayList;
+import java.util.List;
/**
* This is the factory for all ExpressionVisitor instance.
@@ -54,4 +61,46 @@ public class ExpressionVisitors {
return true;
}
}
+
+ /**
+ * Split the expression to equal, range and residual predicate.
+ * Should instance when used.
+ */
+ public static class PredicatesSpliter extends
DefaultExpressionVisitor<Void, Void> {
+
+ private List<Expression> equalPredicates = new ArrayList<>();
+ private List<Expression> rangePredicates = new ArrayList<>();
+ private List<Expression> residualPredicates = new ArrayList<>();
+ private final Expression target;
+
+ public PredicatesSpliter(Expression target) {
+ this.target = target;
+ }
+
+ @Override
+ public Void visitComparisonPredicate(ComparisonPredicate
comparisonPredicate, Void context) {
+ // TODO Smallest implement, complete later
+ if (comparisonPredicate instanceof EqualTo) {
+ Expression leftArgument = comparisonPredicate.getArgument(0);
+ Expression rightArgument = comparisonPredicate.getArgument(1);
+ if (leftArgument.isSlot() && rightArgument.isSlot()) {
+ equalPredicates.add(comparisonPredicate);
+ } else {
+ rangePredicates.add(comparisonPredicate);
+ }
+ }
+ return super.visit(comparisonPredicate, context);
+ }
+
+ public Expression getTarget() {
+ return target;
+ }
+
+ public Predicates.SplitPredicate getSplitPredicate() {
+ return Predicates.SplitPredicate.of(
+ equalPredicates.isEmpty() ? null :
ExpressionUtils.and(equalPredicates),
+ rangePredicates.isEmpty() ? null :
ExpressionUtils.and(rangePredicates),
+ residualPredicates.isEmpty() ? null :
ExpressionUtils.and(residualPredicates));
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
index 0c5faa20957..4793395cfe3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/util/ExpressionUtils.java
@@ -17,7 +17,9 @@
package org.apache.doris.nereids.util;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.nereids.CascadesContext;
+import org.apache.doris.nereids.rules.exploration.mv.SlotMapping;
import org.apache.doris.nereids.rules.expression.ExpressionRewriteContext;
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRule;
import org.apache.doris.nereids.trees.TreeNode;
@@ -39,6 +41,7 @@ import
org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
import
org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter;
+import org.apache.doris.nereids.trees.plans.Plan;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
@@ -196,6 +199,31 @@ public class ExpressionUtils {
.orElse(BooleanLiteral.of(type == And.class));
}
+ /**
+ * Replace the slot in expression with the lineage identifier from
specified
+ * baseTable sets or target table types.
+ * <p>
+ * For example as following:
+ * select a + 10 as a1, d from (
+ * select b - 5 as a, d from table
+ * );
+ * after shuttle a1, d in select will be b - 5 + 10, d
+ */
+ public static List<? extends Expression>
shuttleExpressionWithLineage(List<? extends Expression> expression,
+ Plan plan,
+ Set<TableType> targetTypes,
+ Set<String> tableIdentifiers) {
+ return ImmutableList.of();
+ }
+
+ /**
+ * Replace the slot in expressions according to the slotMapping
+ * if any slot cannot be mapped then return null
+ */
+ public static List<? extends Expression> permute(List<? extends
Expression> expressions, SlotMapping slotMapping) {
+ return ImmutableList.of();
+ }
+
/**
* Choose the minimum slot from input parameter.
*/
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index 00432fd8af6..c07b8df6c19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -451,6 +451,12 @@ public class SessionVariable implements Serializable,
Writable {
public static final String TABLE_STATS_HEALTH_THRESHOLD
= "table_stats_health_threshold";
+ public static final String ENABLE_MATERIALIZED_VIEW_REWRITE
+ = "enable_materialized_view_rewrite";
+
+ public static final String
MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_FOREIGN_TABLE
+ = "materialized_view_rewrite_enable_contain_foreign_table";
+
public static final List<String> DEBUG_VARIABLES = ImmutableList.of(
SKIP_DELETE_PREDICATE,
SKIP_DELETE_BITMAP,
@@ -1369,6 +1375,17 @@ public class SessionVariable implements Serializable,
Writable {
+ "considered outdated."})
public int tableStatsHealthThreshold = 60;
+ @VariableMgr.VarAttr(name = ENABLE_MATERIALIZED_VIEW_REWRITE, needForward
= true,
+ description = {"是否开启基于结构信息的物化视图透明改写",
+ "Whether to enable materialized view rewriting based on
struct info"})
+ public boolean enableMaterializedViewRewrite = false;
+
+ @VariableMgr.VarAttr(name =
MATERIALIZED_VIEW_REWRITE_ENABLE_CONTAIN_FOREIGN_TABLE, needForward = true,
+ description = {"基于结构信息的透明改写，是否使用包含外表的物化视图",
+ "whether to use a materialized view that contains the
foreign table "
+ + "when using rewriting based on struct info"})
+ public boolean materializedViewRewriteEnableContainForeignTable = false;
+
public static final String IGNORE_RUNTIME_FILTER_IDS =
"ignore_runtime_filter_ids";
public Set<Integer> getIgnoredRuntimeFilterIds() {
@@ -2956,4 +2973,12 @@ public class SessionVariable implements Serializable,
Writable {
public boolean isEnableInsertGroupCommit() {
return enableInsertGroupCommit ||
Config.wait_internal_group_commit_finish;
}
+
+ public boolean isEnableMaterializedViewRewrite() {
+ return enableMaterializedViewRewrite;
+ }
+
+ public boolean isMaterializedViewRewriteEnableContainForeignTable() {
+ return materializedViewRewriteEnableContainForeignTable;
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
