This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a change to branch 2.1-tmp
in repository https://gitbox.apache.org/repos/asf/doris.git
from 6c5dd820c0a [improvement](spill) improve spill timers (#33156)
new ff990eb8693 [enhancement](Nereids) refactor expression rewriter to
pattern match (#32617)
new 38d580dfb79 [fix](Nereids) fix link children failed (#33134)
new cc363f26c26 [fix](Nereids) fix group concat (#33091)
new 9bc7902e5a0 [fix](Nereids) fix bind group by int literal (#33117)
The 4 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/pipeline/pipeline_fragment_context.cpp | 9 +-
.../pipeline_x/pipeline_x_fragment_context.cpp | 11 +-
be/src/runtime/descriptors.h | 5 +
be/src/vec/exec/vaggregation_node.h | 1 +
fe/fe-core/pom.xml | 2 +-
.../org/apache/doris/analysis/DateLiteral.java | 9 +-
.../java/org/apache/doris/catalog/OlapTable.java | 5 +-
.../org/apache/doris/mtmv/MTMVRelationManager.java | 4 +-
.../org/apache/doris/mysql/privilege/Role.java | 4 +-
.../org/apache/doris/nereids/CascadesContext.java | 49 +++-
.../org/apache/doris/nereids/NereidsPlanner.java | 2 +-
.../org/apache/doris/nereids/StatementContext.java | 15 ++
.../org/apache/doris/nereids/analyzer/Scope.java | 19 +-
.../java/org/apache/doris/nereids/jobs/Job.java | 11 +-
.../doris/nereids/jobs/executor/Analyzer.java | 2 -
.../doris/nereids/jobs/executor/Rewriter.java | 7 +-
.../jobs/joinorder/hypergraph/HyperGraph.java | 11 +-
.../nereids/jobs/rewrite/CustomRewriteJob.java | 6 +-
.../jobs/rewrite/PlanTreeRewriteBottomUpJob.java | 113 ++++++----
.../nereids/jobs/rewrite/PlanTreeRewriteJob.java | 65 ++++--
.../jobs/rewrite/PlanTreeRewriteTopDownJob.java | 41 +++-
.../nereids/jobs/rewrite/RewriteJobContext.java | 10 +-
.../jobs/rewrite/RootPlanTreeRewriteJob.java | 16 +-
.../java/org/apache/doris/nereids/memo/Group.java | 22 +-
.../apache/doris/nereids/memo/GroupExpression.java | 5 +
.../nereids/pattern/ExpressionPatternRules.java | 112 +++++++++
.../ExpressionPatternTraverseListeners.java | 112 +++++++++
.../doris/nereids/pattern/ParentTypeIdMapping.java | 59 +++++
.../org/apache/doris/nereids/pattern/Pattern.java | 4 +
.../apache/doris/nereids/pattern/TypeMappings.java | 133 +++++++++++
.../generator/ExpressionTypeMappingGenerator.java | 159 +++++++++++++
...GeneratorAnalyzer.java => JavaAstAnalyzer.java} | 93 ++++----
.../generator/LogicalBinaryPatternGenerator.java | 4 +-
.../generator/LogicalLeafPatternGenerator.java | 4 +-
.../generator/LogicalUnaryPatternGenerator.java | 4 +-
.../generator/PatternDescribableProcessor.java | 34 ++-
.../generator/PhysicalBinaryPatternGenerator.java | 4 +-
.../generator/PhysicalLeafPatternGenerator.java | 4 +-
.../generator/PhysicalUnaryPatternGenerator.java | 4 +-
...ernGenerator.java => PlanPatternGenerator.java} | 18 +-
.../generator/PlanPatternGeneratorAnalyzer.java | 73 ++++++
.../generator/PlanTypeMappingGenerator.java | 159 +++++++++++++
.../processor/post/RuntimeFilterPruner.java | 17 +-
.../doris/nereids/processor/post/Validator.java | 10 +-
.../properties/ChildrenPropertiesRegulator.java | 4 +-
.../nereids/properties/FunctionalDependencies.java | 24 +-
.../nereids/properties/LogicalProperties.java | 50 +++--
.../nereids/properties/PhysicalProperties.java | 6 +-
.../java/org/apache/doris/nereids/rules/Rule.java | 6 +-
.../org/apache/doris/nereids/rules/RuleSet.java | 4 +-
.../AdjustAggregateNullableForEmptySet.java | 29 ++-
.../nereids/rules/analysis/BindExpression.java | 61 +++--
.../nereids/rules/analysis/BindSlotWithPaths.java | 29 +--
.../nereids/rules/analysis/CheckAfterRewrite.java | 85 +++----
.../nereids/rules/analysis/CheckAnalysis.java | 36 +--
.../rules/analysis/EliminateGroupByConstant.java | 2 +-
.../nereids/rules/analysis/ExpressionAnalyzer.java | 2 +-
.../nereids/rules/analysis/FillUpMissingSlots.java | 21 +-
.../nereids/rules/analysis/NormalizeAggregate.java | 48 ++--
.../analysis/ReplaceExpressionByChildOutput.java | 48 ++--
.../ResolveOrdinalInOrderByAndGroupBy.java | 102 ---------
.../nereids/rules/analysis/SubqueryToApply.java | 77 +++++--
.../mv/AbstractMaterializedViewRule.java | 15 +-
.../mv/InitMaterializationContextHook.java | 4 +-
.../exploration/mv/MaterializationContext.java | 8 +-
.../nereids/rules/exploration/mv/StructInfo.java | 20 +-
.../expression/ExpressionBottomUpRewriter.java | 124 ++++++++++
.../expression/ExpressionListenerMatcher.java | 41 ++++
...eContext.java => ExpressionMatchingAction.java} | 16 +-
...Context.java => ExpressionMatchingContext.java} | 25 ++-
.../rules/expression/ExpressionNormalization.java | 29 +--
...=> ExpressionNormalizationAndOptimization.java} | 22 +-
.../rules/expression/ExpressionOptimization.java | 26 ++-
.../expression/ExpressionPatternMatchRule.java | 64 ++++++
.../rules/expression/ExpressionPatternMatcher.java | 41 ++++
.../expression/ExpressionPatternRuleFactory.java | 84 +++++++
.../rules/expression/ExpressionRewrite.java | 51 ++++-
.../rules/expression/ExpressionRewriteContext.java | 4 +-
.../rules/expression/ExpressionRuleExecutor.java | 16 +-
...ontext.java => ExpressionTraverseListener.java} | 16 +-
.../ExpressionTraverseListenerFactory.java | 79 +++++++
.../ExpressionTraverseListenerMapping.java | 59 +++++
.../nereids/rules/expression/check/CheckCast.java | 24 +-
.../rules/ArrayContainToArrayOverlap.java | 94 +++++---
.../rules/expression/rules/CaseWhenToIf.java | 18 +-
.../expression/rules/ConvertAggStateCast.java | 33 ++-
.../expression/rules/DateFunctionRewrite.java | 34 ++-
.../expression/rules/DigitalMaskingConvert.java | 23 +-
.../expression/rules/DistinctPredicatesRule.java | 18 +-
.../expression/rules/ExtractCommonFactorRule.java | 222 +++++++++++++++---
.../rules/expression/rules/FoldConstantRule.java | 32 ++-
.../expression/rules/FoldConstantRuleOnBE.java | 46 ++--
.../expression/rules/FoldConstantRuleOnFE.java | 170 ++++++++++++--
.../rules/expression/rules/InPredicateDedup.java | 40 ++--
.../expression/rules/InPredicateToEqualToRule.java | 25 ++-
.../rules/NormalizeBinaryPredicatesRule.java | 21 +-
.../expression/rules/NullSafeEqualToEqual.java | 21 +-
.../rules/OneListPartitionEvaluator.java | 2 +-
.../rules/OneRangePartitionEvaluator.java | 120 ++++++----
.../nereids/rules/expression/rules/OrToIn.java | 36 +--
.../rules/expression/rules/PartitionPruner.java | 23 +-
.../expression/rules/PartitionRangeExpander.java | 115 ++++++----
.../rules/PredicateRewriteForPartitionPrune.java | 4 +-
.../rules/RangePartitionValueIterator.java | 64 ++++++
.../expression/rules/ReplaceVariableByLiteral.java | 17 +-
.../rules/SimplifyArithmeticComparisonRule.java | 105 +++++----
.../expression/rules/SimplifyArithmeticRule.java | 70 +++---
.../rules/expression/rules/SimplifyCastRule.java | 21 +-
.../rules/SimplifyComparisonPredicate.java | 37 ++-
.../rules/SimplifyDecimalV3Comparison.java | 24 +-
.../expression/rules/SimplifyInPredicate.java | 20 +-
.../expression/rules/SimplifyNotExprRule.java | 34 ++-
.../rules/expression/rules/SimplifyRange.java | 73 +++---
.../expression/rules/SupportJavaDateFormatter.java | 44 ++--
.../nereids/rules/expression/rules/TopnToMax.java | 29 ++-
.../rules/TryEliminateUninterestedPredicates.java | 14 +-
.../rules/implementation/AggregateStrategies.java | 34 ++-
.../rules/rewrite/AdjustConjunctsReturnType.java | 4 +-
.../nereids/rules/rewrite/AdjustNullable.java | 12 +-
.../rules/rewrite/CheckMatchExpression.java | 7 +-
.../nereids/rules/rewrite/CheckPrivileges.java | 29 ++-
.../doris/nereids/rules/rewrite/ColumnPruning.java | 98 ++++----
.../rules/rewrite/CountDistinctRewrite.java | 60 ++---
.../nereids/rules/rewrite/CountLiteralRewrite.java | 37 ++-
.../nereids/rules/rewrite/EliminateFilter.java | 7 +-
.../nereids/rules/rewrite/EliminateGroupBy.java | 56 +++--
.../nereids/rules/rewrite/EliminateMarkJoin.java | 17 +-
.../nereids/rules/rewrite/EliminateNotNull.java | 39 ++--
.../rules/rewrite/EliminateOrderByConstant.java | 16 +-
.../ExtractAndNormalizeWindowExpression.java | 161 +++++++------
...xtractSingleTableExpressionFromDisjunction.java | 9 +-
.../nereids/rules/rewrite/InferJoinNotNull.java | 4 +-
.../nereids/rules/rewrite/MergeAggregate.java | 2 +-
.../doris/nereids/rules/rewrite/MergeProjects.java | 10 +-
.../doris/nereids/rules/rewrite/NormalizeSort.java | 59 +++--
.../nereids/rules/rewrite/NormalizeToSlot.java | 43 ++--
.../rules/rewrite/PruneOlapScanPartition.java | 51 +++--
.../nereids/rules/rewrite/PullUpPredicates.java | 75 ++++---
.../rewrite/PushDownFilterThroughAggregation.java | 12 +-
.../rewrite/PushDownFilterThroughProject.java | 13 +-
.../doris/nereids/rules/rewrite/ReorderJoin.java | 6 +-
.../nereids/rules/rewrite/SimplifyAggGroupBy.java | 23 +-
.../mv/AbstractSelectMaterializedIndexRule.java | 15 +-
.../mv/SelectMaterializedIndexWithAggregate.java | 9 +-
.../SelectMaterializedIndexWithoutAggregate.java | 45 ++--
.../doris/nereids/stats/StatsCalculator.java | 11 +-
.../doris/nereids/trees/AbstractTreeNode.java | 22 +-
.../org/apache/doris/nereids/trees/TreeNode.java | 17 ++
.../nereids/trees/expressions/BinaryOperator.java | 6 -
.../trees/expressions/ComparisonPredicate.java | 4 +-
.../nereids/trees/expressions/Expression.java | 99 +++++---
.../nereids/trees/expressions/InPredicate.java | 5 +-
.../nereids/trees/expressions/SlotReference.java | 7 +-
.../functions/ComputeSignatureHelper.java | 11 +-
.../functions/agg/AggregateFunction.java | 17 +-
.../scalar/PushDownToProjectionFunction.java | 7 +-
.../trees/expressions/literal/DateLiteral.java | 39 +++-
.../visitor/DefaultExpressionRewriter.java | 10 +-
.../doris/nereids/trees/plans/AbstractPlan.java | 28 +--
.../org/apache/doris/nereids/trees/plans/Plan.java | 61 +++--
.../nereids/trees/plans/algebra/Aggregate.java | 17 +-
.../doris/nereids/trees/plans/algebra/Project.java | 27 ++-
.../trees/plans/logical/LogicalAggregate.java | 8 +-
.../plans/logical/LogicalCatalogRelation.java | 132 ++++++-----
.../trees/plans/logical/LogicalOlapScan.java | 65 ++++--
.../trees/plans/logical/LogicalProject.java | 8 +-
.../nereids/trees/plans/logical/LogicalSort.java | 19 +-
.../nereids/trees/plans/logical/LogicalTopN.java | 13 +-
.../trees/plans/physical/PhysicalHashJoin.java | 3 +-
.../apache/doris/nereids/util/ExpressionUtils.java | 250 ++++++++++++++++-----
.../doris/nereids/util/ImmutableEqualSet.java | 6 +-
.../org/apache/doris/nereids/util/JoinUtils.java | 5 +-
.../org/apache/doris/nereids/util/PlanUtils.java | 24 ++
.../doris/nereids/util/TypeCoercionUtils.java | 19 +-
.../java/org/apache/doris/nereids/util/Utils.java | 65 +++++-
.../java/org/apache/doris/qe/SessionVariable.java | 38 +++-
.../jobs/joinorder/hypergraph/HyperGraphTest.java | 12 +-
.../rules/expression/ExpressionRewriteTest.java | 80 +++++--
.../expression/ExpressionRewriteTestHelper.java | 2 +-
.../nereids/rules/expression/FoldConstantTest.java | 36 ++-
.../rules/expression/PredicatesSplitterTest.java | 2 +-
.../expression/SimplifyArithmeticRuleTest.java | 56 +++--
.../rules/expression/SimplifyInPredicateTest.java | 8 +-
.../rules/expression/SimplifyRangeTest.java | 26 ++-
.../expression/rules/NullSafeEqualToEqualTest.java | 20 +-
.../SimplifyArithmeticComparisonRuleTest.java | 7 +-
.../expression/rules/SimplifyCastRuleTest.java | 7 +-
.../rules/SimplifyComparisonPredicateTest.java | 35 ++-
.../rules/SimplifyDecimalV3ComparisonTest.java | 6 +-
.../rules/expression/rules/TopnToMaxTest.java | 4 +-
.../rules/rewrite/EliminateJoinByFkTest.java | 1 +
.../doris/nereids/rules/rewrite/OrToInTest.java | 19 +-
.../PushDownFilterThroughAggregationTest.java | 4 +-
.../functions/ComputeSignatureHelperTest.java | 11 +
.../nereids/trees/plans/SetOperationTest.java | 38 ++++
.../org/apache/doris/nereids/util/PlanChecker.java | 20 ++
.../data/nereids_hint_tpcds_p0/shape/query24.out | 10 +-
.../data/nereids_hint_tpcds_p0/shape/query64.out | 2 +-
.../filter_push_down/push_filter_through.out | 28 +--
.../data/nereids_syntax_p0/bind_priority.out | 6 +
.../shape/query13.out | 2 +-
.../shape/query14.out | 2 +-
.../shape/query24.out | 4 +-
.../shape/query41.out | 2 +-
.../shape/query50.out | 2 +-
.../shape/query64.out | 2 +-
.../shape/query85.out | 2 +-
.../shape/query95.out | 2 +-
.../noStatsRfPrune/query13.out | 2 +-
.../noStatsRfPrune/query17.out | 2 +-
.../noStatsRfPrune/query41.out | 2 +-
.../noStatsRfPrune/query47.out | 7 +-
.../noStatsRfPrune/query50.out | 2 +-
.../noStatsRfPrune/query57.out | 7 +-
.../noStatsRfPrune/query6.out | 57 +++--
.../noStatsRfPrune/query65.out | 2 +-
.../no_stats_shape/query13.out | 2 +-
.../no_stats_shape/query14.out | 2 +-
.../no_stats_shape/query17.out | 2 +-
.../no_stats_shape/query24.out | 2 +-
.../no_stats_shape/query41.out | 2 +-
.../no_stats_shape/query47.out | 7 +-
.../no_stats_shape/query50.out | 2 +-
.../no_stats_shape/query57.out | 7 +-
.../no_stats_shape/query6.out | 57 +++--
.../no_stats_shape/query64.out | 2 +-
.../no_stats_shape/query65.out | 2 +-
.../no_stats_shape/query85.out | 6 +-
.../rf_prune/query13.out | 2 +-
.../rf_prune/query14.out | 2 +-
.../rf_prune/query41.out | 2 +-
.../rf_prune/query50.out | 2 +-
.../rf_prune/query85.out | 2 +-
.../rf_prune/query95.out | 2 +-
.../nereids_tpcds_shape_sf100_p0/shape/query13.out | 2 +-
.../nereids_tpcds_shape_sf100_p0/shape/query14.out | 2 +-
.../nereids_tpcds_shape_sf100_p0/shape/query24.out | 4 +-
.../nereids_tpcds_shape_sf100_p0/shape/query41.out | 2 +-
.../nereids_tpcds_shape_sf100_p0/shape/query50.out | 2 +-
.../nereids_tpcds_shape_sf100_p0/shape/query64.out | 2 +-
.../nereids_tpcds_shape_sf100_p0/shape/query85.out | 2 +-
.../nereids_tpcds_shape_sf100_p0/shape/query95.out | 2 +-
.../data/nereids_tpch_shape_sf1000_p0/shape/q9.out | 2 +-
.../shape_no_stats/q9.out | 2 +-
.../org/apache/doris/regression/suite/Suite.groovy | 7 +-
.../doris/regression/util/OutputUtils.groovy | 28 ++-
.../doris/regression/util/ReusableIterator.groovy | 7 +
.../suites/nereids_syntax_p0/bind_priority.groovy | 28 +++
248 files changed, 5153 insertions(+), 2081 deletions(-)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternRules.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ExpressionPatternTraverseListeners.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/ParentTypeIdMapping.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/TypeMappings.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/ExpressionTypeMappingGenerator.java
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/{PatternGeneratorAnalyzer.java
=> JavaAstAnalyzer.java} (75%)
rename
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/{PatternGenerator.java
=> PlanPatternGenerator.java} (96%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PlanPatternGeneratorAnalyzer.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/pattern/generator/PlanTypeMappingGenerator.java
delete mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/ResolveOrdinalInOrderByAndGroupBy.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionBottomUpRewriter.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionListenerMatcher.java
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/{ExpressionRewriteContext.java
=> ExpressionMatchingAction.java} (73%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/{ExpressionRewriteContext.java
=> ExpressionMatchingContext.java} (55%)
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/{ExpressionRewriteContext.java
=> ExpressionNormalizationAndOptimization.java} (59%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionPatternMatchRule.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionPatternMatcher.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionPatternRuleFactory.java
copy
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/{ExpressionRewriteContext.java
=> ExpressionTraverseListener.java} (66%)
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionTraverseListenerFactory.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/ExpressionTraverseListenerMapping.java
create mode 100644
fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RangePartitionValueIterator.java
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
