This is an automated email from the ASF dual-hosted git repository.
huajianlan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 5858beea761 [refactor](nereids) refactor analyze view (#37106)
5858beea761 is described below
commit 5858beea761b55380c81b2f39fa0ba8868fdeeaf
Author: 924060929 <924060...@qq.com>
AuthorDate: Tue Jul 2 19:47:03 2024 +0800
[refactor](nereids) refactor analyze view (#37106)
The Analyzer of NereidsPlanner use different rules to analyze normal plan
and view, to prevent the plans in the views analyze multiple times, because
some rules can not apply multiple times, say, decimal type coercion, if this
rule applied multiple times, it will generate a wrong result.
But this design is trick. Normally, after process the LogicalView, the
whole plan tree in the LogicalView should not contains unbound plan, but the
current situation is not like this. And this problem block development of some
rules, so I refactor it:
1. the Analyzer will not traverse the children of the LogicalView
2. After link the LogicalView to the outer plan tree, the whole plan tree
of the LogicalView will not contains unbound plan
3. analyze view and table use the same rules, keep it simple
---
.../org/apache/doris/nereids/CascadesContext.java | 12 +-
.../jobs/executor/AbstractBatchJobExecutor.java | 40 +-
.../doris/nereids/jobs/executor/Analyzer.java | 80 +--
.../doris/nereids/jobs/executor/Rewriter.java | 663 +++++++++++----------
.../jobs/rewrite/PlanTreeRewriteBottomUpJob.java | 22 +-
.../nereids/jobs/rewrite/PlanTreeRewriteJob.java | 6 +-
.../jobs/rewrite/PlanTreeRewriteTopDownJob.java | 22 +-
.../jobs/rewrite/RootPlanTreeRewriteJob.java | 13 +-
.../doris/nereids/rules/analysis/BindRelation.java | 2 +-
9 files changed, 454 insertions(+), 406 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
index 3b9ba912383..a0d748c08c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java
@@ -240,19 +240,11 @@ public class CascadesContext implements ScheduleContext {
}
public Analyzer newAnalyzer() {
- return newAnalyzer(false);
- }
-
- public Analyzer newAnalyzer(boolean analyzeView) {
- return new Analyzer(this, analyzeView);
- }
-
- public Analyzer newAnalyzer(boolean analyzeView,
Optional<CustomTableResolver> customTableResolver) {
- return new Analyzer(this, analyzeView, customTableResolver);
+ return newAnalyzer(Optional.empty());
}
public Analyzer newAnalyzer(Optional<CustomTableResolver>
customTableResolver) {
- return newAnalyzer(false, customTableResolver);
+ return new Analyzer(this, customTableResolver);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
index bec86debc9e..4eebf6ffc05 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
@@ -29,6 +29,7 @@ import org.apache.doris.nereids.jobs.rewrite.TopicRewriteJob;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleFactory;
import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter;
import com.google.common.collect.ImmutableList;
@@ -36,6 +37,8 @@ import com.google.common.collect.ImmutableList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
+import java.util.Set;
+import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -46,6 +49,8 @@ import java.util.stream.Stream;
* Each batch of rules will be uniformly executed.
*/
public abstract class AbstractBatchJobExecutor {
+ private static final ThreadLocal<Set<Class<Plan>>> NOT_TRAVERSE_CHILDREN =
new ThreadLocal();
+ private static final Predicate<Plan> TRAVERSE_ALL_PLANS = plan -> true;
protected CascadesContext cascadesContext;
@@ -65,6 +70,17 @@ public abstract class AbstractBatchJobExecutor {
).collect(ImmutableList.toImmutableList());
}
+ /** notTraverseChildrenOf */
+ public static <T> T notTraverseChildrenOf(
+ Set<Class<? extends Plan>> notTraverseClasses, Supplier<T> action)
{
+ try {
+ NOT_TRAVERSE_CHILDREN.set((Set) notTraverseClasses);
+ return action.get();
+ } finally {
+ NOT_TRAVERSE_CHILDREN.remove();
+ }
+ }
+
public static TopicRewriteJob topic(String topicName, RewriteJob... jobs) {
return new TopicRewriteJob(topicName, Arrays.asList(jobs));
}
@@ -82,7 +98,7 @@ public abstract class AbstractBatchJobExecutor {
.map(RuleFactory::buildRules)
.flatMap(List::stream)
.collect(ImmutableList.toImmutableList());
- return new RootPlanTreeRewriteJob(rules,
PlanTreeRewriteBottomUpJob::new, true);
+ return new RootPlanTreeRewriteJob(rules,
PlanTreeRewriteBottomUpJob::new, getTraversePredicate(), true);
}
public static RewriteJob topDown(RuleFactory... ruleFactories) {
@@ -98,7 +114,7 @@ public abstract class AbstractBatchJobExecutor {
.map(RuleFactory::buildRules)
.flatMap(List::stream)
.collect(ImmutableList.toImmutableList());
- return new RootPlanTreeRewriteJob(rules,
PlanTreeRewriteTopDownJob::new, once);
+ return new RootPlanTreeRewriteJob(rules,
PlanTreeRewriteTopDownJob::new, getTraversePredicate(), once);
}
public static RewriteJob custom(RuleType ruleType,
Supplier<CustomRewriter> planRewriter) {
@@ -126,4 +142,24 @@ public abstract class AbstractBatchJobExecutor {
}
public abstract List<RewriteJob> getJobs();
+
+ private static Predicate<Plan> getTraversePredicate() {
+ Set<Class<Plan>> notTraverseChildren = NOT_TRAVERSE_CHILDREN.get();
+ return notTraverseChildren == null
+ ? TRAVERSE_ALL_PLANS
+ : new NotTraverseChildren(notTraverseChildren);
+ }
+
+ private static class NotTraverseChildren implements Predicate<Plan> {
+ private final Set<Class<Plan>> notTraverseChildren;
+
+ public NotTraverseChildren(Set<Class<Plan>> notTraverseChildren) {
+ this.notTraverseChildren =
Objects.requireNonNull(notTraverseChildren, "notTraversePlans can not be null");
+ }
+
+ @Override
+ public boolean test(Plan plan) {
+ return !notTraverseChildren.contains(plan.getClass());
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
index b72240cb8e5..4c840dd69d3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java
@@ -48,6 +48,10 @@ import
org.apache.doris.nereids.rules.analysis.VariableToLiteral;
import org.apache.doris.nereids.rules.rewrite.MergeProjects;
import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute;
import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
+import org.apache.doris.nereids.trees.plans.logical.LogicalView;
+
+import com.google.common.collect.ImmutableSet;
import java.util.List;
import java.util.Objects;
@@ -59,8 +63,7 @@ import java.util.Optional;
*/
public class Analyzer extends AbstractBatchJobExecutor {
- public static final List<RewriteJob> DEFAULT_ANALYZE_JOBS =
buildAnalyzeJobs(Optional.empty());
- public static final List<RewriteJob> DEFAULT_ANALYZE_VIEW_JOBS =
buildAnalyzeViewJobs(Optional.empty());
+ public static final List<RewriteJob> ANALYZE_JOBS =
buildAnalyzeJobs(Optional.empty());
private final List<RewriteJob> jobs;
@@ -69,36 +72,23 @@ public class Analyzer extends AbstractBatchJobExecutor {
* @param cascadesContext planner context for execute job
*/
public Analyzer(CascadesContext cascadesContext) {
- this(cascadesContext, false);
- }
-
- public Analyzer(CascadesContext cascadesContext, boolean analyzeView) {
- this(cascadesContext, analyzeView, Optional.empty());
+ this(cascadesContext, Optional.empty());
}
/**
* constructor of Analyzer. For view, we only do bind relation since other
analyze step will do by outer Analyzer.
*
* @param cascadesContext current context for analyzer
- * @param analyzeView analyze view or user sql. If true, analyzer is used
for view.
* @param customTableResolver custom resolver for outer catalog.
*/
- public Analyzer(CascadesContext cascadesContext, boolean analyzeView,
- Optional<CustomTableResolver> customTableResolver) {
+ public Analyzer(CascadesContext cascadesContext,
Optional<CustomTableResolver> customTableResolver) {
super(cascadesContext);
Objects.requireNonNull(customTableResolver, "customTableResolver
cannot be null");
- if (analyzeView) {
- if (customTableResolver.isPresent()) {
- this.jobs = buildAnalyzeViewJobs(customTableResolver);
- } else {
- this.jobs = DEFAULT_ANALYZE_VIEW_JOBS;
- }
+
+ if (customTableResolver.isPresent()) {
+ this.jobs = buildAnalyzeJobs(customTableResolver);
} else {
- if (customTableResolver.isPresent()) {
- this.jobs = buildAnalyzeJobs(customTableResolver);
- } else {
- this.jobs = DEFAULT_ANALYZE_JOBS;
- }
+ this.jobs = ANALYZE_JOBS;
}
}
@@ -114,47 +104,43 @@ public class Analyzer extends AbstractBatchJobExecutor {
execute();
}
- private static List<RewriteJob>
buildAnalyzeViewJobs(Optional<CustomTableResolver> customTableResolver) {
- return jobs(
- topDown(new AnalyzeCTE()),
- topDown(new EliminateLogicalSelectHint()),
- bottomUp(
- new BindRelation(customTableResolver),
- new CheckPolicy()
- )
+ private static List<RewriteJob>
buildAnalyzeJobs(Optional<CustomTableResolver> customTableResolver) {
+ return notTraverseChildrenOf(
+ ImmutableSet.of(LogicalView.class, LogicalCTEAnchor.class),
+ () -> buildAnalyzerJobs(customTableResolver)
);
}
- private static List<RewriteJob>
buildAnalyzeJobs(Optional<CustomTableResolver> customTableResolver) {
+ private static List<RewriteJob>
buildAnalyzerJobs(Optional<CustomTableResolver> customTableResolver) {
return jobs(
// we should eliminate hint before "Subquery unnesting".
topDown(new AnalyzeCTE()),
topDown(new EliminateLogicalSelectHint()),
bottomUp(
- new BindRelation(customTableResolver),
- new CheckPolicy()
+ new BindRelation(customTableResolver),
+ new CheckPolicy()
),
bottomUp(new BindExpression()),
topDown(new BindSink()),
bottomUp(new CheckAfterBind()),
bottomUp(
- new ProjectToGlobalAggregate(),
- // this rule check's the logicalProject node's isDistinct
property
- // and replace the logicalProject node with a LogicalAggregate
node
- // so any rule before this, if create a new logicalProject node
- // should make sure isDistinct property is correctly passed
around.
- // please see rule BindSlotReference or BindFunction for
example
- new EliminateDistinctConstant(),
- new ProjectWithDistinctToAggregate(),
- new ReplaceExpressionByChildOutput(),
- new OneRowRelationExtractAggregate()
+ new ProjectToGlobalAggregate(),
+ // this rule check's the logicalProject node's isDistinct
property
+ // and replace the logicalProject node with a
LogicalAggregate node
+ // so any rule before this, if create a new logicalProject
node
+ // should make sure isDistinct property is correctly
passed around.
+ // please see rule BindSlotReference or BindFunction for
example
+ new EliminateDistinctConstant(),
+ new ProjectWithDistinctToAggregate(),
+ new ReplaceExpressionByChildOutput(),
+ new OneRowRelationExtractAggregate()
),
topDown(
- new FillUpMissingSlots(),
- // We should use NormalizeRepeat to compute nullable
properties for LogicalRepeat in the analysis
- // stage. NormalizeRepeat will compute nullable property, add
virtual slot, LogicalAggregate and
- // LogicalProject for normalize. This rule depends on
FillUpMissingSlots to fill up slots.
- new NormalizeRepeat()
+ new FillUpMissingSlots(),
+ // We should use NormalizeRepeat to compute nullable
properties for LogicalRepeat in the analysis
+ // stage. NormalizeRepeat will compute nullable property,
add virtual slot, LogicalAggregate and
+ // LogicalProject for normalize. This rule depends on
FillUpMissingSlots to fill up slots.
+ new NormalizeRepeat()
),
bottomUp(new AdjustAggregateNullableForEmptySet()),
// consider sql with user defined var @t_zone
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 0a2906ca055..9f3ea9bc378 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -142,8 +142,10 @@ import
org.apache.doris.nereids.rules.rewrite.batch.CorrelateApplyToUnCorrelateA
import
org.apache.doris.nereids.rules.rewrite.batch.EliminateUselessPlanUnderApply;
import
org.apache.doris.nereids.rules.rewrite.mv.SelectMaterializedIndexWithAggregate;
import
org.apache.doris.nereids.rules.rewrite.mv.SelectMaterializedIndexWithoutAggregate;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import com.google.common.collect.ImmutableList;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import java.util.List;
@@ -154,306 +156,315 @@ import java.util.stream.Collectors;
*/
public class Rewriter extends AbstractBatchJobExecutor {
- private static final List<RewriteJob>
CTE_CHILDREN_REWRITE_JOBS_BEFORE_SUB_PATH_PUSH_DOWN = jobs(
- topic("Plan Normalization",
- topDown(
- new EliminateOrderByConstant(),
- new EliminateSortUnderSubqueryOrView(),
- new EliminateGroupByConstant(),
- // MergeProjects depends on this rule
- new LogicalSubQueryAliasToLogicalProject(),
- // TODO: we should do expression normalization
after plan normalization
- // because some rewritten depends on sub
expression tree matching
- // such as group by key matching and replaced
- // but we need to do some normalization before
subquery unnesting,
- // such as extract common expression.
- new ExpressionNormalizationAndOptimization(),
- new AvgDistinctToSumDivCount(),
- new CountDistinctRewrite(),
- new ExtractFilterFromCrossJoin()
- ),
- topDown(
- // ExtractSingleTableExpressionFromDisjunction
conflict to InPredicateToEqualToRule
- // in the ExpressionNormalization, so must invoke
in another job, otherwise dead loop.
- new ExtractSingleTableExpressionFromDisjunction()
- )
- ),
- // subquery unnesting relay on ExpressionNormalization to extract
common factor expression
- topic("Subquery unnesting",
- // after doing NormalizeAggregate in analysis job
- // we need run the following 2 rules to make
AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION work
- bottomUp(new PullUpProjectUnderApply()),
- topDown(new PushDownFilterThroughProject()),
- custom(RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION,
- AggScalarSubQueryToWindowFunction::new),
- bottomUp(
- new EliminateUselessPlanUnderApply(),
- // CorrelateApplyToUnCorrelateApply and ApplyToJoin
- // and SelectMaterializedIndexWithAggregate
depends on this rule
- new MergeProjects(),
- /*
- * Subquery unnesting.
- * 1. Adjust the plan in correlated logicalApply
- * so that there are no correlated columns in
the subquery.
- * 2. Convert logicalApply to a logicalJoin.
- * TODO: group these rules to make sure the
result plan is what we expected.
- */
- new CorrelateApplyToUnCorrelateApply(),
- new ApplyToJoin()
- )
- ),
- // before `Subquery unnesting` topic, some correlate slots should
have appeared at LogicalApply.left,
- // but it appeared at LogicalApply.right. After the `Subquery
unnesting` topic, all slots is placed in a
- // normal position, then we can check column privileges by these
steps
- //
- // 1. use ColumnPruning rule to derive the used slots in
LogicalView
- // 2. and then check the column privileges
- // 3. finally, we can eliminate the LogicalView
- topic("Inline view and check column privileges",
- custom(RuleType.CHECK_PRIVILEGES, CheckPrivileges::new),
- bottomUp(new InlineLogicalView())
- ),
- topic("Eliminate optimization",
- bottomUp(
- new EliminateLimit(),
- new EliminateFilter(),
- new EliminateAggregate(),
- new EliminateAggCaseWhen(),
- new ReduceAggregateChildOutputRows(),
- new EliminateJoinCondition(),
- new EliminateAssertNumRows(),
- new EliminateSemiJoin()
- )
- ),
- // please note: this rule must run before NormalizeAggregate
- topDown(new AdjustAggregateNullableForEmptySet()),
- // The rule modification needs to be done after the subquery is
unnested,
- // because for scalarSubQuery, the connection condition is stored
in apply in the analyzer phase,
- // but when normalizeAggregate/normalizeSort is performed, the
members in apply cannot be obtained,
- // resulting in inconsistent output results and results in apply
- topDown(
- new NormalizeAggregate(),
- new CountLiteralRewrite(),
- new NormalizeSort()
- ),
- topic("Window analysis",
- topDown(
- new ExtractAndNormalizeWindowExpression(),
- new CheckAndStandardizeWindowFunctionAndFrame(),
- new SimplifyWindowExpression()
- )
- ),
- topic("Rewrite join",
- // infer not null filter, then push down filter, and then
reorder join(cross join to inner join)
- topDown(
- new InferAggNotNull(),
- new InferFilterNotNull(),
- new InferJoinNotNull()
- ),
- // ReorderJoin depends PUSH_DOWN_FILTERS
- // the PUSH_DOWN_FILTERS depends on lots of rules, e.g.
merge project, eliminate outer,
- // sometimes transform the bottom plan make some rules
usable which can apply to the top plan,
- // but top-down traverse can not cover this case in one
iteration, so bottom-up is more
- // efficient because it can find the new plans and apply
transform wherever it is
- bottomUp(RuleSet.PUSH_DOWN_FILTERS),
- // after push down, some new filters are generated, which
needs to be optimized. (example: tpch q19)
- // topDown(new ExpressionOptimization()),
- topDown(
- new MergeFilters(),
- new ReorderJoin(),
- new PushFilterInsideJoin(),
- new FindHashConditionForJoin(),
- new ConvertInnerOrCrossJoin(),
- new EliminateNullAwareLeftAntiJoin()
- ),
- // push down SEMI Join
- bottomUp(
- new TransposeSemiJoinLogicalJoin(),
- new TransposeSemiJoinLogicalJoinProject(),
- new TransposeSemiJoinAgg(),
- new TransposeSemiJoinAggProject()
- ),
- topDown(
- new EliminateDedupJoinCondition()
- ),
- // eliminate useless not null or inferred not null
- // TODO: wait InferPredicates to infer more not null.
- bottomUp(new EliminateNotNull()),
- topDown(new ConvertInnerOrCrossJoin()),
- topDown(new ProjectOtherJoinConditionForNestedLoopJoin())
- ),
- topic("Column pruning and infer predicate",
- custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
- custom(RuleType.INFER_PREDICATES, InferPredicates::new),
- // column pruning create new project, so we should use
PUSH_DOWN_FILTERS
- // to change filter-project to project-filter
- bottomUp(RuleSet.PUSH_DOWN_FILTERS),
- // after eliminate outer join in the PUSH_DOWN_FILTERS, we
can infer more predicate and push down
- custom(RuleType.INFER_PREDICATES, InferPredicates::new),
- bottomUp(RuleSet.PUSH_DOWN_FILTERS),
- // after eliminate outer join, we can move some filters to
join.otherJoinConjuncts,
- // this can help to translate plan to backend
- topDown(new PushFilterInsideJoin()),
- topDown(new FindHashConditionForJoin()),
- topDown(new ExpressionNormalization())
- ),
-
- // this rule should invoke after ColumnPruning
- custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new),
-
- topic("Set operation optimization",
- // Do MergeSetOperation first because we hope to match
pattern of Distinct SetOperator.
- topDown(new PushProjectThroughUnion(), new
MergeProjects()),
- bottomUp(new MergeSetOperations(), new
MergeSetOperationsExcept()),
- bottomUp(new PushProjectIntoOneRowRelation()),
- topDown(new MergeOneRowRelationIntoUnion()),
- topDown(new PushProjectIntoUnion()),
- costBased(topDown(new InferSetOperatorDistinct())),
- topDown(new BuildAggForUnion())
- ),
+ private static final List<RewriteJob>
CTE_CHILDREN_REWRITE_JOBS_BEFORE_SUB_PATH_PUSH_DOWN = notTraverseChildrenOf(
+ ImmutableSet.of(LogicalCTEAnchor.class),
+ () -> jobs(
+ topic("Plan Normalization",
+ topDown(
+ new EliminateOrderByConstant(),
+ new EliminateSortUnderSubqueryOrView(),
+ new EliminateGroupByConstant(),
+ // MergeProjects depends on this rule
+ new LogicalSubQueryAliasToLogicalProject(),
+ // TODO: we should do expression normalization
after plan normalization
+ // because some rewritten depends on sub
expression tree matching
+ // such as group by key matching and replaced
+ // but we need to do some normalization
before subquery unnesting,
+ // such as extract common expression.
+ new ExpressionNormalizationAndOptimization(),
+ new AvgDistinctToSumDivCount(),
+ new CountDistinctRewrite(),
+ new ExtractFilterFromCrossJoin()
+ ),
+ topDown(
+ // ExtractSingleTableExpressionFromDisjunction
conflict to InPredicateToEqualToRule
+ // in the ExpressionNormalization, so must
invoke in another job, otherwise dead loop.
+ new
ExtractSingleTableExpressionFromDisjunction()
+ )
+ ),
+ // subquery unnesting relay on ExpressionNormalization to
extract common factor expression
+ topic("Subquery unnesting",
+ // after doing NormalizeAggregate in analysis job
+ // we need run the following 2 rules to make
AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION work
+ bottomUp(new PullUpProjectUnderApply()),
+ topDown(new PushDownFilterThroughProject()),
+ custom(RuleType.AGG_SCALAR_SUBQUERY_TO_WINDOW_FUNCTION,
+ AggScalarSubQueryToWindowFunction::new),
+ bottomUp(
+ new EliminateUselessPlanUnderApply(),
+ // CorrelateApplyToUnCorrelateApply and
ApplyToJoin
+ // and SelectMaterializedIndexWithAggregate
depends on this rule
+ new MergeProjects(),
+ /*
+ * Subquery unnesting.
+ * 1. Adjust the plan in correlated
logicalApply
+ * so that there are no correlated columns
in the subquery.
+ * 2. Convert logicalApply to a logicalJoin.
+ * TODO: group these rules to make sure the
result plan is what we expected.
+ */
+ new CorrelateApplyToUnCorrelateApply(),
+ new ApplyToJoin()
+ )
+ ),
+ // before `Subquery unnesting` topic, some correlate slots
should have appeared at LogicalApply.left,
+ // but it appeared at LogicalApply.right. After the `Subquery
unnesting` topic, all slots is placed in a
+ // normal position, then we can check column privileges by
these steps
+ //
+ // 1. use ColumnPruning rule to derive the used slots in
LogicalView
+ // 2. and then check the column privileges
+ // 3. finally, we can eliminate the LogicalView
+ topic("Inline view and check column privileges",
+ custom(RuleType.CHECK_PRIVILEGES,
CheckPrivileges::new),
+ bottomUp(new InlineLogicalView())
+ ),
+ topic("Eliminate optimization",
+ bottomUp(
+ new EliminateLimit(),
+ new EliminateFilter(),
+ new EliminateAggregate(),
+ new EliminateAggCaseWhen(),
+ new ReduceAggregateChildOutputRows(),
+ new EliminateJoinCondition(),
+ new EliminateAssertNumRows(),
+ new EliminateSemiJoin()
+ )
+ ),
+ // please note: this rule must run before NormalizeAggregate
+ topDown(new AdjustAggregateNullableForEmptySet()),
+ // The rule modification needs to be done after the subquery
is unnested,
+ // because for scalarSubQuery, the connection condition is
stored in apply in the analyzer phase,
+ // but when normalizeAggregate/normalizeSort is performed, the
members in apply cannot be obtained,
+ // resulting in inconsistent output results and results in
apply
+ topDown(
+ new NormalizeAggregate(),
+ new CountLiteralRewrite(),
+ new NormalizeSort()
+ ),
+ topic("Window analysis",
+ topDown(
+ new ExtractAndNormalizeWindowExpression(),
+ new
CheckAndStandardizeWindowFunctionAndFrame(),
+ new SimplifyWindowExpression()
+ )
+ ),
+ topic("Rewrite join",
+ // infer not null filter, then push down filter, and
then reorder join(cross join to inner join)
+ topDown(
+ new InferAggNotNull(),
+ new InferFilterNotNull(),
+ new InferJoinNotNull()
+ ),
+ // ReorderJoin depends PUSH_DOWN_FILTERS
+ // the PUSH_DOWN_FILTERS depends on lots of rules,
e.g. merge project, eliminate outer,
+ // sometimes transform the bottom plan make some rules
usable which can apply to the top plan,
+ // but top-down traverse can not cover this case in
one iteration, so bottom-up is more
+ // efficient because it can find the new plans and
apply transform wherever it is
+ bottomUp(RuleSet.PUSH_DOWN_FILTERS),
+ // after push down, some new filters are generated,
which needs to be optimized.
+ // (example: tpch q19)
+ // topDown(new ExpressionOptimization()),
+ topDown(
+ new MergeFilters(),
+ new ReorderJoin(),
+ new PushFilterInsideJoin(),
+ new FindHashConditionForJoin(),
+ new ConvertInnerOrCrossJoin(),
+ new EliminateNullAwareLeftAntiJoin()
+ ),
+ // push down SEMI Join
+ bottomUp(
+ new TransposeSemiJoinLogicalJoin(),
+ new TransposeSemiJoinLogicalJoinProject(),
+ new TransposeSemiJoinAgg(),
+ new TransposeSemiJoinAggProject()
+ ),
+ topDown(
+ new EliminateDedupJoinCondition()
+ ),
+ // eliminate useless not null or inferred not null
+ // TODO: wait InferPredicates to infer more not null.
+ bottomUp(new EliminateNotNull()),
+ topDown(new ConvertInnerOrCrossJoin()),
+ topDown(new
ProjectOtherJoinConditionForNestedLoopJoin())
+ ),
+ topic("Column pruning and infer predicate",
+ custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
+ custom(RuleType.INFER_PREDICATES,
InferPredicates::new),
+ // column pruning create new project, so we should use
PUSH_DOWN_FILTERS
+ // to change filter-project to project-filter
+ bottomUp(RuleSet.PUSH_DOWN_FILTERS),
+ // after eliminate outer join in the PUSH_DOWN_FILTERS,
+ // we can infer more predicate and push down
+ custom(RuleType.INFER_PREDICATES,
InferPredicates::new),
+ bottomUp(RuleSet.PUSH_DOWN_FILTERS),
+ // after eliminate outer join, we can move some
filters to join.otherJoinConjuncts,
+ // this can help to translate plan to backend
+ topDown(new PushFilterInsideJoin()),
+ topDown(new FindHashConditionForJoin()),
+ topDown(new ExpressionNormalization())
+ ),
- topic("Eliminate GroupBy",
- topDown(new EliminateGroupBy(),
- new MergeAggregate(),
- // need to adjust min/max/sum nullable attribute
after merge aggregate
- new AdjustAggregateNullableForEmptySet())
- ),
+ // this rule should invoke after ColumnPruning
+ custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new),
- topic("Eager aggregation",
- costBased(topDown(
- new PushDownAggThroughJoinOneSide(),
- new PushDownAggThroughJoin()
- )),
- costBased(custom(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN,
PushDownDistinctThroughJoin::new))
- ),
+ topic("Set operation optimization",
+ // Do MergeSetOperation first because we hope to match
pattern of Distinct SetOperator.
+ topDown(new PushProjectThroughUnion(), new
MergeProjects()),
+ bottomUp(new MergeSetOperations(), new
MergeSetOperationsExcept()),
+ bottomUp(new PushProjectIntoOneRowRelation()),
+ topDown(new MergeOneRowRelationIntoUnion()),
+ topDown(new PushProjectIntoUnion()),
+ costBased(topDown(new InferSetOperatorDistinct())),
+ topDown(new BuildAggForUnion())
+ ),
- // this rule should invoke after infer predicate and push down
distinct, and before push down limit
- topic("eliminate join according unique or foreign key",
- bottomUp(new EliminateJoinByFK()),
- topDown(new EliminateJoinByUnique())
- ),
+ topic("Eliminate GroupBy",
+ topDown(new EliminateGroupBy(),
+ new MergeAggregate(),
+ // need to adjust min/max/sum nullable
attribute after merge aggregate
+ new AdjustAggregateNullableForEmptySet())
+ ),
- // this rule should be after topic "Column pruning and infer
predicate"
- topic("Join pull up",
- topDown(
- new EliminateFilter(),
- new PushDownFilterThroughProject(),
- new MergeProjects()
- ),
- topDown(
- new PullUpJoinFromUnionAll()
- ),
- custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
- bottomUp(RuleSet.PUSH_DOWN_FILTERS),
- custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new)
- ),
+ topic("Eager aggregation",
+ costBased(topDown(
+ new PushDownAggThroughJoinOneSide(),
+ new PushDownAggThroughJoin()
+ )),
+
costBased(custom(RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN,
PushDownDistinctThroughJoin::new))
+ ),
- // this rule should be invoked after topic "Join pull up"
- topic("eliminate Aggregate according to fd items",
- topDown(new EliminateGroupByKey()),
- topDown(new PushDownAggThroughJoinOnPkFk())
- ),
+ // this rule should invoke after infer predicate and push down
distinct, and before push down limit
+ topic("eliminate join according unique or foreign key",
+ bottomUp(new EliminateJoinByFK()),
+ topDown(new EliminateJoinByUnique())
+ ),
- topic("Limit optimization",
- // TODO: the logical plan should not contains any phase
information,
- // we should refactor like AggregateStrategies, e.g.
LimitStrategies,
- // generate one PhysicalLimit if current
distribution is gather or two
- // PhysicalLimits with gather exchange
- topDown(new LimitSortToTopN()),
- topDown(new MergeTopNs()),
- topDown(new SplitLimit()),
- topDown(
- new PushDownLimit(),
- new PushDownLimitDistinctThroughJoin(),
- new PushDownLimitDistinctThroughUnion(),
- new PushDownTopNDistinctThroughJoin(),
- new PushDownTopNDistinctThroughUnion(),
- new PushDownTopNThroughJoin(),
- new PushDownTopNThroughWindow(),
- new PushDownTopNThroughUnion()
- ),
- topDown(new CreatePartitionTopNFromWindow()),
- topDown(
- new PullUpProjectUnderTopN(),
- new PullUpProjectUnderLimit()
- )
- ),
- // TODO: these rules should be implementation rules, and generate
alternative physical plans.
- topic("Table/Physical optimization",
- topDown(
- new PruneOlapScanPartition(),
- new PruneEmptyPartition(),
- new PruneFileScanPartition(),
- new PushConjunctsIntoJdbcScan(),
- new PushConjunctsIntoOdbcScan(),
- new PushConjunctsIntoEsScan()
- )
- ),
- topic("MV optimization",
- topDown(
- new SelectMaterializedIndexWithAggregate(),
- new SelectMaterializedIndexWithoutAggregate(),
+ // this rule should be after topic "Column pruning and infer
predicate"
+ topic("Join pull up",
+ topDown(
new EliminateFilter(),
new PushDownFilterThroughProject(),
- new MergeProjects(),
- new PruneOlapScanTablet()
- ),
- custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
- bottomUp(RuleSet.PUSH_DOWN_FILTERS),
- custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new)
- ),
- topic("adjust preagg status",
- topDown(new AdjustPreAggStatus())
- ),
- topic("Point query short circuit",
- topDown(new LogicalResultSinkToShortCircuitPointQuery())),
- topic("eliminate",
- // SORT_PRUNING should be applied after mergeLimit
- custom(RuleType.ELIMINATE_SORT, EliminateSort::new),
- bottomUp(new EliminateEmptyRelation())
- ),
- topic("agg rewrite",
- // these rules should be put after mv optimization to avoid mv
matching fail
- topDown(new SumLiteralRewrite(),
- new MergePercentileToArray())
- ),
- topic("Push project and filter on cte consumer to cte producer",
- topDown(
- new CollectFilterAboveConsumer(),
- new CollectCteConsumerOutput()
- )
- ),
- topic("Collect used column", custom(RuleType.COLLECT_COLUMNS,
QueryColumnCollector::new))
+ new MergeProjects()
+ ),
+ topDown(
+ new PullUpJoinFromUnionAll()
+ ),
+ custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
+ bottomUp(RuleSet.PUSH_DOWN_FILTERS),
+ custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new)
+ ),
+
+ // this rule should be invoked after topic "Join pull up"
+ topic("eliminate Aggregate according to fd items",
+ topDown(new EliminateGroupByKey()),
+ topDown(new PushDownAggThroughJoinOnPkFk())
+ ),
+
+ topic("Limit optimization",
+ // TODO: the logical plan should not contains any
phase information,
+ // we should refactor like AggregateStrategies,
e.g. LimitStrategies,
+ // generate one PhysicalLimit if current
distribution is gather or two
+ // PhysicalLimits with gather exchange
+ topDown(new LimitSortToTopN()),
+ topDown(new MergeTopNs()),
+ topDown(new SplitLimit()),
+ topDown(
+ new PushDownLimit(),
+ new PushDownLimitDistinctThroughJoin(),
+ new PushDownLimitDistinctThroughUnion(),
+ new PushDownTopNDistinctThroughJoin(),
+ new PushDownTopNDistinctThroughUnion(),
+ new PushDownTopNThroughJoin(),
+ new PushDownTopNThroughWindow(),
+ new PushDownTopNThroughUnion()
+ ),
+ topDown(new CreatePartitionTopNFromWindow()),
+ topDown(
+ new PullUpProjectUnderTopN(),
+ new PullUpProjectUnderLimit()
+ )
+ ),
+ // TODO: these rules should be implementation rules, and
generate alternative physical plans.
+ topic("Table/Physical optimization",
+ topDown(
+ new PruneOlapScanPartition(),
+ new PruneEmptyPartition(),
+ new PruneFileScanPartition(),
+ new PushConjunctsIntoJdbcScan(),
+ new PushConjunctsIntoOdbcScan(),
+ new PushConjunctsIntoEsScan()
+ )
+ ),
+ topic("MV optimization",
+ topDown(
+ new SelectMaterializedIndexWithAggregate(),
+ new SelectMaterializedIndexWithoutAggregate(),
+ new EliminateFilter(),
+ new PushDownFilterThroughProject(),
+ new MergeProjects(),
+ new PruneOlapScanTablet()
+ ),
+ custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
+ bottomUp(RuleSet.PUSH_DOWN_FILTERS),
+ custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new)
+ ),
+ topic("adjust preagg status",
+ topDown(new AdjustPreAggStatus())
+ ),
+ topic("Point query short circuit",
+ topDown(new
LogicalResultSinkToShortCircuitPointQuery())),
+ topic("eliminate",
+ // SORT_PRUNING should be applied after mergeLimit
+ custom(RuleType.ELIMINATE_SORT, EliminateSort::new),
+ bottomUp(new EliminateEmptyRelation())
+ ),
+ topic("agg rewrite",
+ // these rules should be put after mv optimization to
avoid mv matching fail
+ topDown(new SumLiteralRewrite(),
+ new MergePercentileToArray())
+ ),
+ topic("Push project and filter on cte consumer to cte
producer",
+ topDown(
+ new CollectFilterAboveConsumer(),
+ new CollectCteConsumerOutput()
+ )
+ ),
+ topic("Collect used column", custom(RuleType.COLLECT_COLUMNS,
QueryColumnCollector::new)
+ )
+ )
);
- private static final List<RewriteJob>
CTE_CHILDREN_REWRITE_JOBS_AFTER_SUB_PATH_PUSH_DOWN = jobs(
- // after variant sub path pruning, we need do column pruning again
- custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
- bottomUp(ImmutableList.of(
- new PushDownFilterThroughProject(),
- new MergeProjects()
- )),
- custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new),
- topic("topn optimize",
- topDown(new DeferMaterializeTopNResult())
- ),
- topic("add projection for join",
- custom(RuleType.ADD_PROJECT_FOR_JOIN,
AddProjectForJoin::new),
- topDown(new MergeProjects())
- ),
- // this rule batch must keep at the end of rewrite to do some plan
check
- topic("Final rewrite and check",
- custom(RuleType.CHECK_DATA_TYPES, CheckDataTypes::new),
- topDown(new PushDownFilterThroughProject(), new
MergeProjects()),
- custom(RuleType.ADJUST_CONJUNCTS_RETURN_TYPE,
AdjustConjunctsReturnType::new),
- bottomUp(
- new
ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE),
- new CheckMatchExpression(),
- new CheckMultiDistinct(),
- new CheckAfterRewrite()
- )
- ),
- topDown(new CollectCteConsumerOutput())
+ private static final List<RewriteJob>
CTE_CHILDREN_REWRITE_JOBS_AFTER_SUB_PATH_PUSH_DOWN = notTraverseChildrenOf(
+ ImmutableSet.of(LogicalCTEAnchor.class),
+ () -> jobs(
+ // after variant sub path pruning, we need do column pruning
again
+ custom(RuleType.COLUMN_PRUNING, ColumnPruning::new),
+ bottomUp(ImmutableList.of(
+ new PushDownFilterThroughProject(),
+ new MergeProjects()
+ )),
+ custom(RuleType.ELIMINATE_UNNECESSARY_PROJECT,
EliminateUnnecessaryProject::new),
+ topic("topn optimize",
+ topDown(new DeferMaterializeTopNResult())
+ ),
+ topic("add projection for join",
+ custom(RuleType.ADD_PROJECT_FOR_JOIN,
AddProjectForJoin::new),
+ topDown(new MergeProjects())
+ ),
+ // this rule batch must keep at the end of rewrite to do some
plan check
+ topic("Final rewrite and check",
+ custom(RuleType.CHECK_DATA_TYPES, CheckDataTypes::new),
+ topDown(new PushDownFilterThroughProject(), new
MergeProjects()),
+ custom(RuleType.ADJUST_CONJUNCTS_RETURN_TYPE,
AdjustConjunctsReturnType::new),
+ bottomUp(
+ new
ExpressionRewrite(CheckLegalityAfterRewrite.INSTANCE),
+ new CheckMatchExpression(),
+ new CheckMultiDistinct(),
+ new CheckAfterRewrite()
+ )
+ ),
+ topDown(new CollectCteConsumerOutput())
+ )
);
private static final List<RewriteJob> WHOLE_TREE_REWRITE_JOBS
@@ -503,39 +514,45 @@ public class Rewriter extends AbstractBatchJobExecutor {
List<RewriteJob> beforePushDownJobs,
List<RewriteJob> afterPushDownJobs) {
- List<RewriteJob> rewriteJobs = Lists.newArrayListWithExpectedSize(300);
- rewriteJobs.addAll(jobs(
- topic("cte inline and pull up all cte anchor",
- custom(RuleType.PULL_UP_CTE_ANCHOR,
PullUpCteAnchor::new),
- custom(RuleType.CTE_INLINE, CTEInline::new)
- ),
- topic("process limit session variables",
- custom(RuleType.ADD_DEFAULT_LIMIT,
AddDefaultLimit::new)
- ),
- topic("rewrite cte sub-tree before sub path push down",
- custom(RuleType.REWRITE_CTE_CHILDREN, () -> new
RewriteCteChildren(beforePushDownJobs))
- )));
- if (needOrExpansion) {
- rewriteJobs.addAll(jobs(topic("or expansion",
- custom(RuleType.OR_EXPANSION, () ->
OrExpansion.INSTANCE))));
- }
- if (needSubPathPushDown) {
- rewriteJobs.addAll(jobs(
- topic("variant element_at push down",
- custom(RuleType.VARIANT_SUB_PATH_PRUNING,
VariantSubPathPruning::new)
- )
- ));
- }
- rewriteJobs.addAll(jobs(
- topic("rewrite cte sub-tree after sub path push down",
- custom(RuleType.CLEAR_CONTEXT_STATUS,
ClearContextStatus::new),
- custom(RuleType.REWRITE_CTE_CHILDREN, () -> new
RewriteCteChildren(afterPushDownJobs))
- ),
- topic("whole plan check",
- custom(RuleType.ADJUST_NULLABLE, AdjustNullable::new)
- )
- ));
- return rewriteJobs;
+ return notTraverseChildrenOf(
+ ImmutableSet.of(LogicalCTEAnchor.class),
+ () -> {
+ List<RewriteJob> rewriteJobs =
Lists.newArrayListWithExpectedSize(300);
+
+ rewriteJobs.addAll(jobs(
+ topic("cte inline and pull up all cte anchor",
+ custom(RuleType.PULL_UP_CTE_ANCHOR,
PullUpCteAnchor::new),
+ custom(RuleType.CTE_INLINE, CTEInline::new)
+ ),
+ topic("process limit session variables",
+ custom(RuleType.ADD_DEFAULT_LIMIT,
AddDefaultLimit::new)
+ ),
+ topic("rewrite cte sub-tree before sub path push down",
+ custom(RuleType.REWRITE_CTE_CHILDREN, () ->
new RewriteCteChildren(beforePushDownJobs))
+ )));
+ if (needOrExpansion) {
+ rewriteJobs.addAll(jobs(topic("or expansion",
+ custom(RuleType.OR_EXPANSION, () ->
OrExpansion.INSTANCE))));
+ }
+ if (needSubPathPushDown) {
+ rewriteJobs.addAll(jobs(
+ topic("variant element_at push down",
+ custom(RuleType.VARIANT_SUB_PATH_PRUNING,
VariantSubPathPruning::new)
+ )
+ ));
+ }
+ rewriteJobs.addAll(jobs(
+ topic("rewrite cte sub-tree after sub path push down",
+ custom(RuleType.CLEAR_CONTEXT_STATUS,
ClearContextStatus::new),
+ custom(RuleType.REWRITE_CTE_CHILDREN, () ->
new RewriteCteChildren(afterPushDownJobs))
+ ),
+ topic("whole plan check",
+ custom(RuleType.ADJUST_NULLABLE,
AdjustNullable::new)
+ )
+ ));
+ return rewriteJobs;
+ }
+ );
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java
index 60555a9cc04..aecb03a4158 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteBottomUpJob.java
@@ -21,11 +21,11 @@ import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import java.util.List;
import java.util.Objects;
import java.util.Optional;
+import java.util.function.Predicate;
/**
* PlanTreeRewriteBottomUpJob
@@ -55,8 +55,10 @@ public class PlanTreeRewriteBottomUpJob extends
PlanTreeRewriteJob {
ENSURE_CHILDREN_REWRITTEN
}
- public PlanTreeRewriteBottomUpJob(RewriteJobContext rewriteJobContext,
JobContext context, List<Rule> rules) {
- super(JobType.BOTTOM_UP_REWRITE, context);
+ public PlanTreeRewriteBottomUpJob(
+ RewriteJobContext rewriteJobContext, JobContext context,
+ Predicate<Plan> isTraverseChildren, List<Rule> rules) {
+ super(JobType.BOTTOM_UP_REWRITE, context, isTraverseChildren);
this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext,
"rewriteContext cannot be null");
this.rules = Objects.requireNonNull(rules, "rules cannot be null");
this.batchId = rewriteJobContext.batchId;
@@ -97,7 +99,7 @@ public class PlanTreeRewriteBottomUpJob extends
PlanTreeRewriteJob {
return;
}
// After the rewrite take effect, we should handle the children
part again.
- pushJob(new PlanTreeRewriteBottomUpJob(newJobContext, context,
rules));
+ pushJob(new PlanTreeRewriteBottomUpJob(newJobContext, context,
isTraverseChildren, rules));
setState(rewriteResult.plan,
RewriteState.ENSURE_CHILDREN_REWRITTEN, batchId);
} else {
// No new plan is generated, so just set the state of the current
plan to 'REWRITTEN'.
@@ -110,12 +112,12 @@ public class PlanTreeRewriteBottomUpJob extends
PlanTreeRewriteJob {
Plan plan = rewriteJobContext.plan;
int batchId = rewriteJobContext.batchId;
setState(plan, RewriteState.REWRITE_THIS, batchId);
- pushJob(new PlanTreeRewriteBottomUpJob(rewriteJobContext, context,
rules));
+ pushJob(new PlanTreeRewriteBottomUpJob(rewriteJobContext, context,
isTraverseChildren, rules));
// some rule return new plan tree, which the number of new plan node >
1,
// we should transform this new plan nodes too.
// NOTICE: this relay on pull up cte anchor
- if (!(rewriteJobContext.plan instanceof LogicalCTEAnchor)) {
+ if (isTraverseChildren.test(plan)) {
pushChildrenJobs(plan);
}
}
@@ -128,25 +130,25 @@ public class PlanTreeRewriteBottomUpJob extends
PlanTreeRewriteJob {
Plan child = children.get(0);
RewriteJobContext childRewriteJobContext = new
RewriteJobContext(
child, rewriteJobContext, 0, false, batchId);
- pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext,
context, rules));
+ pushJob(new PlanTreeRewriteBottomUpJob(childRewriteJobContext,
context, isTraverseChildren, rules));
return;
case 2:
Plan right = children.get(1);
RewriteJobContext rightRewriteJobContext = new
RewriteJobContext(
right, rewriteJobContext, 1, false, batchId);
- pushJob(new PlanTreeRewriteBottomUpJob(rightRewriteJobContext,
context, rules));
+ pushJob(new PlanTreeRewriteBottomUpJob(rightRewriteJobContext,
context, isTraverseChildren, rules));
Plan left = children.get(0);
RewriteJobContext leftRewriteJobContext = new
RewriteJobContext(
left, rewriteJobContext, 0, false, batchId);
- pushJob(new PlanTreeRewriteBottomUpJob(leftRewriteJobContext,
context, rules));
+ pushJob(new PlanTreeRewriteBottomUpJob(leftRewriteJobContext,
context, isTraverseChildren, rules));
return;
default:
for (int i = children.size() - 1; i >= 0; i--) {
child = children.get(i);
childRewriteJobContext = new RewriteJobContext(
child, rewriteJobContext, i, false, batchId);
- pushJob(new
PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, rules));
+ pushJob(new
PlanTreeRewriteBottomUpJob(childRewriteJobContext, context, isTraverseChildren,
rules));
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java
index c2b136c40fa..0f87a745b5e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteJob.java
@@ -31,12 +31,16 @@ import org.apache.doris.nereids.trees.plans.Plan;
import com.google.common.collect.ImmutableList;
import java.util.List;
+import java.util.Objects;
+import java.util.function.Predicate;
/** PlanTreeRewriteJob */
public abstract class PlanTreeRewriteJob extends Job {
+ protected final Predicate<Plan> isTraverseChildren;
- public PlanTreeRewriteJob(JobType type, JobContext context) {
+ public PlanTreeRewriteJob(JobType type, JobContext context,
Predicate<Plan> isTraverseChildren) {
super(type, context);
+ this.isTraverseChildren = Objects.requireNonNull(isTraverseChildren,
"isTraverseChildren can not be null");
}
protected final RewriteResult rewrite(Plan plan, List<Rule> rules,
RewriteJobContext rewriteJobContext) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java
index 14019bc885e..19d92e2f4f3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/PlanTreeRewriteTopDownJob.java
@@ -21,10 +21,10 @@ import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.JobType;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
import java.util.List;
import java.util.Objects;
+import java.util.function.Predicate;
/**
* PlanTreeRewriteTopDownJob
@@ -36,8 +36,10 @@ public class PlanTreeRewriteTopDownJob extends
PlanTreeRewriteJob {
private final RewriteJobContext rewriteJobContext;
private final List<Rule> rules;
- public PlanTreeRewriteTopDownJob(RewriteJobContext rewriteJobContext,
JobContext context, List<Rule> rules) {
- super(JobType.TOP_DOWN_REWRITE, context);
+ public PlanTreeRewriteTopDownJob(
+ RewriteJobContext rewriteJobContext, JobContext context,
+ Predicate<Plan> isTraverseChildren, List<Rule> rules) {
+ super(JobType.TOP_DOWN_REWRITE, context, isTraverseChildren);
this.rewriteJobContext = Objects.requireNonNull(rewriteJobContext,
"rewriteContext cannot be null");
this.rules = Objects.requireNonNull(rules, "rules cannot be null");
}
@@ -49,15 +51,15 @@ public class PlanTreeRewriteTopDownJob extends
PlanTreeRewriteJob {
if (rewriteResult.hasNewPlan) {
RewriteJobContext newContext = rewriteJobContext
.withPlanAndChildrenVisited(rewriteResult.plan, false);
- pushJob(new PlanTreeRewriteTopDownJob(newContext, context,
rules));
+ pushJob(new PlanTreeRewriteTopDownJob(newContext, context,
isTraverseChildren, rules));
return;
}
RewriteJobContext newRewriteJobContext =
rewriteJobContext.withChildrenVisited(true);
- pushJob(new PlanTreeRewriteTopDownJob(newRewriteJobContext,
context, rules));
+ pushJob(new PlanTreeRewriteTopDownJob(newRewriteJobContext,
context, isTraverseChildren, rules));
// NOTICE: this relay on pull up cte anchor
- if (!(this.rewriteJobContext.plan instanceof LogicalCTEAnchor)) {
+ if (isTraverseChildren.test(rewriteJobContext.plan)) {
pushChildrenJobs(newRewriteJobContext);
}
} else {
@@ -77,22 +79,22 @@ public class PlanTreeRewriteTopDownJob extends
PlanTreeRewriteJob {
case 1:
RewriteJobContext childRewriteJobContext = new
RewriteJobContext(
children.get(0), rewriteJobContext, 0, false,
this.rewriteJobContext.batchId);
- pushJob(new PlanTreeRewriteTopDownJob(childRewriteJobContext,
context, rules));
+ pushJob(new PlanTreeRewriteTopDownJob(childRewriteJobContext,
context, isTraverseChildren, rules));
return;
case 2:
RewriteJobContext rightRewriteJobContext = new
RewriteJobContext(
children.get(1), rewriteJobContext, 1, false,
this.rewriteJobContext.batchId);
- pushJob(new PlanTreeRewriteTopDownJob(rightRewriteJobContext,
context, rules));
+ pushJob(new PlanTreeRewriteTopDownJob(rightRewriteJobContext,
context, isTraverseChildren, rules));
RewriteJobContext leftRewriteJobContext = new
RewriteJobContext(
children.get(0), rewriteJobContext, 0, false,
this.rewriteJobContext.batchId);
- pushJob(new PlanTreeRewriteTopDownJob(leftRewriteJobContext,
context, rules));
+ pushJob(new PlanTreeRewriteTopDownJob(leftRewriteJobContext,
context, isTraverseChildren, rules));
return;
default:
for (int i = children.size() - 1; i >= 0; i--) {
childRewriteJobContext = new RewriteJobContext(
children.get(i), rewriteJobContext, i, false,
this.rewriteJobContext.batchId);
- pushJob(new
PlanTreeRewriteTopDownJob(childRewriteJobContext, context, rules));
+ pushJob(new
PlanTreeRewriteTopDownJob(childRewriteJobContext, context, isTraverseChildren,
rules));
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java
index 4949d422613..551efa1026d 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/RootPlanTreeRewriteJob.java
@@ -28,6 +28,7 @@ import org.apache.doris.nereids.trees.plans.Plan;
import java.util.List;
import java.util.Objects;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
/** RootPlanTreeRewriteJob */
public class RootPlanTreeRewriteJob implements RewriteJob {
@@ -36,11 +37,18 @@ public class RootPlanTreeRewriteJob implements RewriteJob {
private final List<Rule> rules;
private final RewriteJobBuilder rewriteJobBuilder;
private final boolean once;
+ private final Predicate<Plan> isTraverseChildren;
public RootPlanTreeRewriteJob(List<Rule> rules, RewriteJobBuilder
rewriteJobBuilder, boolean once) {
+ this(rules, rewriteJobBuilder, plan -> true, once);
+ }
+
+ public RootPlanTreeRewriteJob(
+ List<Rule> rules, RewriteJobBuilder rewriteJobBuilder,
Predicate<Plan> isTraverseChildren, boolean once) {
this.rules = Objects.requireNonNull(rules, "rules cannot be null");
this.rewriteJobBuilder = Objects.requireNonNull(rewriteJobBuilder,
"rewriteJobBuilder cannot be null");
this.once = once;
+ this.isTraverseChildren = isTraverseChildren;
}
@Override
@@ -52,7 +60,7 @@ public class RootPlanTreeRewriteJob implements RewriteJob {
int batchId = BATCH_ID.incrementAndGet();
RootRewriteJobContext rewriteJobContext = new RootRewriteJobContext(
root, false, context, batchId);
- Job rewriteJob = rewriteJobBuilder.build(rewriteJobContext, context,
rules);
+ Job rewriteJob = rewriteJobBuilder.build(rewriteJobContext, context,
isTraverseChildren, rules);
context.getScheduleContext().pushJob(rewriteJob);
cascadesContext.getJobScheduler().executeJobPool(cascadesContext);
@@ -67,7 +75,8 @@ public class RootPlanTreeRewriteJob implements RewriteJob {
/** RewriteJobBuilder */
public interface RewriteJobBuilder {
- Job build(RewriteJobContext rewriteJobContext, JobContext jobContext,
List<Rule> rules);
+ Job build(RewriteJobContext rewriteJobContext, JobContext jobContext,
+ Predicate<Plan> isTraverseChildren, List<Rule> rules);
}
/** RootRewriteJobContext */
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
index f92a302bd02..ddb0e199e8c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java
@@ -347,7 +347,7 @@ public class BindRelation extends OneAnalysisRuleFactory {
CascadesContext viewContext = CascadesContext.initContext(
parentContext.getStatementContext(), parsedViewPlan,
PhysicalProperties.ANY);
viewContext.keepOrShowPlanProcess(parentContext.showPlanProcess(), ()
-> {
- viewContext.newAnalyzer(true, customTableResolver).analyze();
+ viewContext.newAnalyzer(customTableResolver).analyze();
});
parentContext.addPlanProcesses(viewContext.getPlanProcesses());
// we should remove all group expression of the plan which in other
memo, so the groupId would not conflict
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
