This is an automated email from the ASF dual-hosted git repository.
morrySnow pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8c01e2ac90c [improvement](eager-agg) Push down distinct aggregation
using statistics (#65099)
8c01e2ac90c is described below
commit 8c01e2ac90c8a8e557f5ad10911cc2b870074cda
Author: feiniaofeiafei <[email protected]>
AuthorDate: Wed Aug 26 17:26:46 2026 +0800
[improvement](eager-agg) Push down distinct aggregation using statistics
(#65099)
### What problem does this PR solve?
Problem Summary: The distinct aggregation pushdown used a separate
cost-based rewrite and supported only one distinct aggregate. Move pure
distinct aggregation into eager aggregation so statistics decide whether
to deduplicate below joins. Support multiple SUM/COUNT distinct
aggregates with identical keys and multi-column COUNT DISTINCT while
keeping the upper distinct aggregate unchanged. Queries containing
non-distinct aggregates or different distinct keys are not rewritten.
### Release note
Use statistics-driven eager aggregation for pure same-key distinct
aggregates across joins.
---
.../apache/doris/nereids/hint/UseCboRuleHint.java | 47 ----
.../jobs/executor/AbstractBatchJobExecutor.java | 19 +-
.../doris/nereids/jobs/executor/Rewriter.java | 18 --
.../nereids/jobs/rewrite/CostBasedRewriteJob.java | 210 -----------------
.../doris/nereids/parser/LogicalPlanBuilder.java | 21 --
.../nereids/properties/SelectHintUseCboRule.java | 53 -----
.../org/apache/doris/nereids/rules/RuleType.java | 4 -
.../rules/analysis/EliminateLogicalSelectHint.java | 13 --
.../rules/rewrite/PushDownAggThroughJoin.java | 242 -------------------
.../PushDownAggWithDistinctThroughJoinOneSide.java | 176 --------------
.../rewrite/eageraggregation/EagerAggRewriter.java | 2 +-
.../eageraggregation/PushDownAggregation.java | 133 +++++++----
.../analysis/EliminateLogicalSelectHintTest.java | 56 -----
.../rewrite/PushDownCountThroughJoinTest.java | 216 -----------------
.../rules/rewrite/PushDownSumThroughJoinTest.java | 134 -----------
.../eageraggregation/EagerAggRewriterTest.java | 85 ++++++-
.../eager_agg/distinct_agg_func_push_down.out | 237 +++++++++++++++++++
.../eager_agg/distinct_agg_func_push_down.groovy | 256 +++++++++++++++++++++
18 files changed, 661 insertions(+), 1261 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseCboRuleHint.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseCboRuleHint.java
deleted file mode 100644
index 34e18ab8568..00000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/hint/UseCboRuleHint.java
+++ /dev/null
@@ -1,47 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.hint;
-
-/**
- * rule hint.
- */
-public class UseCboRuleHint extends Hint {
-
- private final boolean isNotUseCboRule;
-
- public UseCboRuleHint(String hintName, boolean isNotUseCboRule) {
- super(hintName);
- this.isNotUseCboRule = isNotUseCboRule;
- }
-
- public boolean isNotUseCboRule() {
- return isNotUseCboRule;
- }
-
- @Override
- public String getExplainString() {
- StringBuilder out = new StringBuilder();
- if (isNotUseCboRule) {
- out.append("no_use_");
- } else {
- out.append("use_");
- }
- out.append(getHintName());
- return out.toString();
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
index f9c82df2416..8cd266e3b73 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/AbstractBatchJobExecutor.java
@@ -22,7 +22,6 @@ import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.jobs.rewrite.AdaptiveBottomUpRewriteJob;
import org.apache.doris.nereids.jobs.rewrite.AdaptiveTopDownRewriteJob;
import org.apache.doris.nereids.jobs.rewrite.BottomUpVisitorRewriteJob;
-import org.apache.doris.nereids.jobs.rewrite.CostBasedRewriteJob;
import org.apache.doris.nereids.jobs.rewrite.CustomRewriteJob;
import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteBottomUpJob;
import org.apache.doris.nereids.jobs.rewrite.PlanTreeRewriteTopDownJob;
@@ -98,10 +97,6 @@ public abstract class AbstractBatchJobExecutor {
return new TopicRewriteJob(topicName, Arrays.asList(jobs), condition);
}
- public static RewriteJob costBased(RewriteJob... jobs) {
- return new CostBasedRewriteJob(Arrays.asList(jobs));
- }
-
public static RewriteJob bottomUp(RuleFactory... ruleFactories) {
return bottomUp(Arrays.asList(ruleFactories));
}
@@ -161,21 +156,15 @@ public abstract class AbstractBatchJobExecutor {
continue;
}
- if (shouldRun(currentJob, jobContext, jobs, i)) {
- do {
- jobContext.setRewritten(false);
- currentJob.execute(jobContext);
- } while (!currentJob.isOnce() && jobContext.isRewritten());
- }
+ do {
+ jobContext.setRewritten(false);
+ currentJob.execute(jobContext);
+ } while (!currentJob.isOnce() && jobContext.isRewritten());
}
}
public abstract List<RewriteJob> getJobs();
- protected boolean shouldRun(RewriteJob rewriteJob, JobContext jobContext,
List<RewriteJob> jobs, int jobIndex) {
- return true;
- }
-
private static Predicate<Plan> getTraversePredicate() {
Set<Class<Plan>> notTraverseChildren = NOT_TRAVERSE_CHILDREN.get();
return notTraverseChildren == null
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
index 162289c3355..7a3d6495204 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java
@@ -18,8 +18,6 @@
package org.apache.doris.nereids.jobs.executor;
import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.jobs.rewrite.CostBasedRewriteJob;
import org.apache.doris.nereids.jobs.rewrite.RewriteJob;
import org.apache.doris.nereids.rules.RuleSet;
import org.apache.doris.nereids.rules.RuleType;
@@ -131,7 +129,6 @@ import
org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit;
import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN;
import org.apache.doris.nereids.rules.rewrite.PushCountIntoUnionAll;
import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOnPkFk;
-import
org.apache.doris.nereids.rules.rewrite.PushDownAggWithDistinctThroughJoinOneSide;
import org.apache.doris.nereids.rules.rewrite.PushDownEncodeSlot;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterIntoSchemaScan;
import org.apache.doris.nereids.rules.rewrite.PushDownFilterThroughProject;
@@ -686,7 +683,6 @@ public class Rewriter extends AbstractBatchJobExecutor {
cascadesContext ->
cascadesContext.rewritePlanContainsTypes(
LogicalAggregate.class, LogicalJoin.class
),
- costBased(topDown(new
PushDownAggWithDistinctThroughJoinOneSide())),
custom(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN,
PushDownAggregation::new),
topDown(new PushCountIntoUnionAll())
),
@@ -977,20 +973,6 @@ public class Rewriter extends AbstractBatchJobExecutor {
return rewriteJobs;
}
- @Override
- protected boolean shouldRun(RewriteJob rewriteJob, JobContext jobContext,
List<RewriteJob> jobs, int jobIndex) {
- if (rewriteJob instanceof CostBasedRewriteJob) {
- if (runCboRules) {
- jobContext.setRemainJobs(jobs.subList(jobIndex + 1,
jobs.size()));
- return true;
- } else {
- return false;
- }
- } else {
- return true;
- }
- }
-
@Override
public void execute() {
MoreFieldsThread.keepFunctionSignature(() -> {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java
deleted file mode 100644
index ac0df804da2..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/rewrite/CostBasedRewriteJob.java
+++ /dev/null
@@ -1,210 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.jobs.rewrite;
-
-import org.apache.doris.common.Pair;
-import org.apache.doris.nereids.CascadesContext;
-import org.apache.doris.nereids.StatementContext;
-import org.apache.doris.nereids.cost.Cost;
-import org.apache.doris.nereids.hint.Hint;
-import org.apache.doris.nereids.hint.UseCboRuleHint;
-import org.apache.doris.nereids.jobs.JobContext;
-import org.apache.doris.nereids.jobs.executor.Optimizer;
-import org.apache.doris.nereids.jobs.executor.Rewriter;
-import org.apache.doris.nereids.memo.GroupExpression;
-import org.apache.doris.nereids.rules.Rule;
-import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalCTEAnchor;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
-import org.apache.doris.qe.ConnectContext;
-
-import com.google.common.collect.ImmutableList;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.List;
-import java.util.Optional;
-
-/**
- * Cost based rewrite job.
- * This job do
- */
-public class CostBasedRewriteJob implements RewriteJob {
-
- private static final Logger LOG =
LogManager.getLogger(CostBasedRewriteJob.class);
-
- private final List<RewriteJob> rewriteJobs;
-
- public CostBasedRewriteJob(List<RewriteJob> rewriteJobs) {
- this.rewriteJobs = rewriteJobs;
- // need to generate real rewrite job list
- }
-
- @Override
- public void execute(JobContext jobContext) {
- // checkHint.first means whether it use hint and checkHint.second
means what
- // kind of hint it used
- Pair<Boolean, Hint> checkHint = checkRuleHint();
- // this means it no_use_cbo_rule(xxx) hint
- if (checkHint.first && checkHint.second == null) {
- return;
- }
- CascadesContext currentCtx = jobContext.getCascadesContext();
- CascadesContext skipCboRuleCtx =
CascadesContext.newCurrentTreeContext(currentCtx);
- CascadesContext applyCboRuleCtx =
CascadesContext.newCurrentTreeContext(currentCtx);
- // execute cbo rule on one candidate
- Rewriter.getCteChildrenRewriter(applyCboRuleCtx,
rewriteJobs).execute();
- Plan applyCboPlan = applyCboRuleCtx.getRewritePlan();
- if
(skipCboRuleCtx.getRewritePlan().deepEquals(applyCboRuleCtx.getRewritePlan())) {
- // this means rewrite do not do anything
- return;
- }
-
- StatementContext.CteEnvironmentSnapshot cteEnvSnapshot =
currentCtx.getStatementContext().cacheCteEnvironment();
- // compare two candidates
- Optional<Pair<Cost, GroupExpression>> skipCboRuleCost =
getCost(currentCtx, skipCboRuleCtx, jobContext);
- currentCtx.getStatementContext().restoreCteEnvironment(cteEnvSnapshot);
- Optional<Pair<Cost, GroupExpression>> appliedCboRuleCost =
getCost(currentCtx, applyCboRuleCtx, jobContext);
- currentCtx.getStatementContext().restoreCteEnvironment(cteEnvSnapshot);
- // If one of them optimize failed, just return
- if (!skipCboRuleCost.isPresent() || !appliedCboRuleCost.isPresent()) {
- LOG.warn("Cbo rewrite execute failed on sql: {}, jobs are {}, plan
is {}.",
-
currentCtx.getStatementContext().getOriginStatement().originStmt,
- rewriteJobs, currentCtx.getRewritePlan());
- return;
- }
- if (checkHint.first) {
- checkHint.second.setStatus(Hint.HintStatus.SUCCESS);
- if (!((UseCboRuleHint) checkHint.second).isNotUseCboRule()) {
-
currentCtx.addPlanProcesses(applyCboRuleCtx.getPlanProcesses());
- currentCtx.setRewritePlan(applyCboRuleCtx.getRewritePlan());
- }
- return;
- }
- // If the candidate applied cbo rule is better, replace the original
plan with
- // it.
- if (appliedCboRuleCost.get().first.getValue() <
skipCboRuleCost.get().first.getValue()) {
- currentCtx.setRewritePlan(applyCboPlan);
- }
- }
-
- /**
- * check if we have use rule hint or no use rule hint
- * return an optional object which checkHint.first means whether it use
hint
- * and checkHint.second means what kind of hint it used
- * example, when we use *+ no_use_cbo_rule(xxx) * the optional would be
(true,
- * false)
- * which means it use hint and the hint forbid this kind of rule
- */
- private Pair<Boolean, Hint> checkRuleHint() {
- Pair<Boolean, Hint> checkResult = Pair.of(false, null);
- RewriteJob rewriteJob = rewriteJobs.get(0);
- List<Rule> rules = ImmutableList.of();
- if (rewriteJob instanceof AdaptiveTopDownRewriteJob) {
- rules = ((AdaptiveTopDownRewriteJob) rewriteJob).getRules();
- } else if (rewriteJob instanceof AdaptiveBottomUpRewriteJob) {
- rules = ((AdaptiveBottomUpRewriteJob) rewriteJob).getRules();
- } else if (rewriteJob instanceof RootPlanTreeRewriteJob) {
- rules = ((RootPlanTreeRewriteJob) rewriteJob).getRules();
- } else if (rewriteJob instanceof TopDownVisitorRewriteJob) {
- rules = ((TopDownVisitorRewriteJob)
rewriteJob).getRules().getAllRules();
- } else if (rewriteJob instanceof BottomUpVisitorRewriteJob) {
- rules = ((BottomUpVisitorRewriteJob)
rewriteJob).getRules().getAllRules();
- }
- for (Rule rule : rules) {
- checkResult = checkRuleHintWithHintName(rule.getRuleType());
- if (checkResult.first) {
- return checkResult;
- }
- }
- if (rewriteJob instanceof CustomRewriteJob) {
- checkResult = checkRuleHintWithHintName(((CustomRewriteJob)
rewriteJob).getRuleType());
- }
- return checkResult;
- }
-
- /**
- * for these rules we need use_cbo_rule hint to enable it, otherwise it
would be
- * close by default
- */
- private static boolean checkBlackList(RuleType ruleType) {
- List<RuleType> ruleWhiteList = new ArrayList<>(Arrays.asList(
- RuleType.PUSH_DOWN_AGG_THROUGH_JOIN,
- RuleType.PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE,
- RuleType.PUSH_DOWN_DISTINCT_THROUGH_JOIN,
- RuleType.PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE));
- if (!ruleWhiteList.isEmpty() && ruleWhiteList.contains(ruleType)) {
- return true;
- }
- return false;
- }
-
- /**
- * main mechanism of checkRuleHint
- * return an optional object which checkHint.first means whether it use
hint
- * and checkHint.second means what kind of hint it used
- */
- private Pair<Boolean, Hint> checkRuleHintWithHintName(RuleType ruleType) {
- for (Hint hint :
ConnectContext.get().getStatementContext().getHints()) {
- if (hint.getHintName().equalsIgnoreCase(ruleType.name())) {
- return Pair.of(true, hint);
- }
- }
- if (checkBlackList(ruleType)) {
- return Pair.of(true, null);
- }
- return Pair.of(false, null);
- }
-
- @Override
- public boolean isOnce() {
- // TODO: currently, we do not support execute it more than once.
- return true;
- }
-
- private Optional<Pair<Cost, GroupExpression>> getCost(CascadesContext
currentCtx,
- CascadesContext cboCtx, JobContext jobContext) {
- // Do subtree rewriter
- Rewriter.getCteChildrenRewriter(cboCtx,
jobContext.getRemainJobs()).execute();
- CascadesContext rootCtx = currentCtx.getRoot();
- if (rootCtx.getRewritePlan() instanceof LogicalCTEAnchor) {
- // set subtree rewrite cache
- currentCtx.getStatementContext().getRewrittenCteProducer()
- .put(currentCtx.getCurrentTree().orElse(null),
(LogicalPlan) cboCtx.getRewritePlan());
- // Do post tree rewrite
- CascadesContext rootCtxCopy =
CascadesContext.newCurrentTreeContext(rootCtx);
- rootCtxCopy.withPlanProcess(currentCtx.showPlanProcess(), () -> {
-
Rewriter.getWholeTreeRewriterWithoutCostBasedJobs(rootCtxCopy).execute();
- });
- // Do optimize
- new Optimizer(rootCtxCopy).execute();
- return rootCtxCopy.getMemo().getRoot().getLowestCostPlan(
-
rootCtxCopy.getCurrentJobContext().getRequiredProperties());
- } else {
- // Do post tree rewrite
- CascadesContext cboCtxCopy =
CascadesContext.newCurrentTreeContext(cboCtx);
- // Do optimize
- new Optimizer(cboCtxCopy).execute();
- return cboCtxCopy.getMemo().getRoot().getLowestCostPlan(
- cboCtxCopy.getCurrentJobContext().getRequiredProperties());
- }
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
index 69d2a45d4f5..1fc86341310 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java
@@ -522,7 +522,6 @@ import org.apache.doris.nereids.properties.SelectHint;
import org.apache.doris.nereids.properties.SelectHintLeading;
import org.apache.doris.nereids.properties.SelectHintOrdered;
import org.apache.doris.nereids.properties.SelectHintSetVar;
-import org.apache.doris.nereids.properties.SelectHintUseCboRule;
import org.apache.doris.nereids.properties.SelectHintUseMv;
import org.apache.doris.nereids.trees.TableSample;
import org.apache.doris.nereids.trees.expressions.Add;
@@ -4858,26 +4857,6 @@ public class LogicalPlanBuilder extends
DorisParserBaseVisitor<Object> {
case "ordered":
hints.add(new SelectHintOrdered(hintName));
break;
- case "use_cbo_rule":
- List<String> useRuleParameters = new ArrayList<>();
- for (HintAssignmentContext kv :
hintStatement.parameters) {
- if (kv.key != null) {
- String parameterName =
visitIdentifierOrText(kv.key);
- useRuleParameters.add(parameterName);
- }
- }
- hints.add(new SelectHintUseCboRule(hintName,
useRuleParameters, false));
- break;
- case "no_use_cbo_rule":
- List<String> noUseRuleParameters = new
ArrayList<>();
- for (HintAssignmentContext kv :
hintStatement.parameters) {
- String parameterName =
visitIdentifierOrText(kv.key);
- if (kv.key != null) {
- noUseRuleParameters.add(parameterName);
- }
- }
- hints.add(new SelectHintUseCboRule(hintName,
noUseRuleParameters, true));
- break;
default:
break;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseCboRule.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseCboRule.java
deleted file mode 100644
index 47c1ef9dcd7..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/properties/SelectHintUseCboRule.java
+++ /dev/null
@@ -1,53 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.properties;
-
-import java.util.List;
-
-/**
- * select hint CostBasedRule.
- */
-public class SelectHintUseCboRule extends SelectHint {
- private final List<String> parameters;
-
- private final boolean isNotUseCboRule;
-
- public SelectHintUseCboRule(String hintName, List<String> parameters,
boolean isNotUseCboRule) {
- super(hintName);
- this.parameters = parameters;
- this.isNotUseCboRule = isNotUseCboRule;
- }
-
- public List<String> getParameters() {
- return parameters;
- }
-
- public boolean isNotUseCboRule() {
- return isNotUseCboRule;
- }
-
- @Override
- public String getHintName() {
- return super.getHintName();
- }
-
- @Override
- public String toString() {
- return super.getHintName();
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
index 4a57e891695..a981dc49681 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java
@@ -227,8 +227,6 @@ public enum RuleType {
PUSH_DOWN_FILTER_THROUGH_CTE_ANCHOR(RuleTypeClass.REWRITE),
PUSH_DOWN_MAX_MIN_FILTER(RuleTypeClass.REWRITE),
- PUSH_DOWN_DISTINCT_THROUGH_JOIN(RuleTypeClass.REWRITE),
-
ADD_PROJECT_FOR_JOIN(RuleTypeClass.REWRITE),
ADD_PROJECT_FOR_VOLATILE_EXPRESSION(RuleTypeClass.REWRITE),
VARIANT_SUB_PATH_PRUNING(RuleTypeClass.REWRITE),
@@ -238,8 +236,6 @@ public enum RuleType {
COLUMN_PRUNING(RuleTypeClass.REWRITE),
ELIMINATE_SORT(RuleTypeClass.REWRITE),
- PUSH_DOWN_AGG_THROUGH_JOIN_ONE_SIDE(RuleTypeClass.REWRITE),
- PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE(RuleTypeClass.REWRITE),
PUSH_DOWN_AGG_THROUGH_JOIN(RuleTypeClass.REWRITE),
PUSH_DOWN_AGG_THROUGH_JOIN_ON_PKFK(RuleTypeClass.REWRITE),
TRANSPOSE_LOGICAL_SEMI_JOIN_LOGICAL_JOIN(RuleTypeClass.REWRITE),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
index 49db25ca097..b732aecc423 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHint.java
@@ -22,12 +22,10 @@ import org.apache.doris.nereids.StatementContext;
import org.apache.doris.nereids.hint.Hint;
import org.apache.doris.nereids.hint.LeadingHint;
import org.apache.doris.nereids.hint.OrderedHint;
-import org.apache.doris.nereids.hint.UseCboRuleHint;
import org.apache.doris.nereids.hint.UseMvHint;
import org.apache.doris.nereids.properties.SelectHint;
import org.apache.doris.nereids.properties.SelectHintLeading;
import org.apache.doris.nereids.properties.SelectHintSetVar;
-import org.apache.doris.nereids.properties.SelectHintUseCboRule;
import org.apache.doris.nereids.properties.SelectHintUseMv;
import org.apache.doris.nereids.rules.Rule;
import org.apache.doris.nereids.rules.RuleType;
@@ -66,9 +64,6 @@ public class EliminateLogicalSelectHint extends
OneRewriteRuleFactory {
} else if (hintName.equalsIgnoreCase("LEADING")) {
extractLeading((SelectHintLeading) hint,
ctx.cascadesContext,
ctx.statementContext, selectHintPlan);
- } else if (hintName.equalsIgnoreCase("USE_CBO_RULE")
- || hintName.equalsIgnoreCase("NO_USE_CBO_RULE")) {
- extractRule((SelectHintUseCboRule) hint,
ctx.statementContext);
} else if (hintName.equalsIgnoreCase("USE_MV")) {
extractMv((SelectHintUseMv) hint,
ConnectContext.get().getStatementContext());
} else if (hintName.equalsIgnoreCase("NO_USE_MV")) {
@@ -116,14 +111,6 @@ public class EliminateLogicalSelectHint extends
OneRewriteRuleFactory {
}
}
- private void extractRule(SelectHintUseCboRule selectHint, StatementContext
statementContext) {
- // rule hint need added to statementContext only cause it's set in all
scopes
- for (String parameter : selectHint.getParameters()) {
- UseCboRuleHint hint = new UseCboRuleHint(parameter,
selectHint.isNotUseCboRule());
- statementContext.addHint(hint);
- }
- }
-
private void extractMv(SelectHintUseMv selectHint, StatementContext
statementContext) {
boolean isAllMv = selectHint.getTables().isEmpty();
UseMvHint useMvHint = new UseMvHint(selectHint.getHintName(),
selectHint.getTables(),
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoin.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoin.java
deleted file mode 100644
index d92168c90f0..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggThroughJoin.java
+++ /dev/null
@@ -1,242 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.rules.rewrite;
-
-import org.apache.doris.nereids.rules.Rule;
-import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.expressions.Alias;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.Multiply;
-import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
-import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
-import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
-import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
-import org.apache.doris.nereids.util.TypeCoercionUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableList.Builder;
-
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-/**
- * TODO: distinct | just push one level
- * Support Pushdown Count(*)/Count(col).
- * Count(col) -> Sum( cnt * cntStar )
- * Count(*) -> Sum( leftCntStar * rightCntStar )
- * <p>
- * Related paper "Eager aggregation and lazy aggregation".
- * <pre>
- * aggregate: count(x)
- * |
- * join
- * | \
- * | *
- * (x)
- * ->
- * aggregate: Sum( cnt * cntStar )
- * |
- * join
- * | \
- * | aggregate: count(*) as cntStar
- * aggregate: count(x) as cnt
- * </pre>
- * Notice: rule can't optimize condition that groupby is empty when Count(*)
exists.
- */
-public class PushDownAggThroughJoin implements RewriteRuleFactory {
- @Override
- public List<Rule> buildRules() {
- return ImmutableList.of(
- logicalAggregate(
- innerLogicalJoin()
- .when(join -> join.getOtherJoinConjuncts().isEmpty())
- .whenNot(join -> join.children().stream().anyMatch(p
-> p instanceof LogicalAggregate))
- )
- .when(agg ->
agg.getGroupByExpressions().stream().allMatch(e -> e instanceof Slot))
- .when(agg -> {
- Set<AggregateFunction> funcs =
agg.getAggregateFunctions();
- return !funcs.isEmpty() && funcs.stream()
- .allMatch(f -> !f.isDistinct()
- && (f instanceof Count &&
(((Count) f).isCountStar() || f.child(
- 0) instanceof Slot)
- || (f instanceof Sum && f.child(0)
instanceof Slot))
- );
- })
- .thenApply(ctx -> {
- LogicalAggregate<LogicalJoin<Plan, Plan>> agg =
ctx.root;
- return pushAgg(agg, agg.child(),
ImmutableList.of());
- })
- .toRule(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN),
- logicalAggregate(
- logicalProject(
- innerLogicalJoin()
- .when(join ->
join.getOtherJoinConjuncts().isEmpty())
- .whenNot(join ->
join.children().stream()
- .anyMatch(p -> p instanceof
LogicalAggregate))
- ).when(LogicalProject::isAllSlots)
- )
- .when(agg -> agg.getGroupByExpressions().stream()
- .allMatch(e -> e instanceof Slot))
- .when(agg -> {
- Set<AggregateFunction> funcs =
agg.getAggregateFunctions();
- return !funcs.isEmpty() && funcs.stream()
- .allMatch(f -> !f.isDistinct()
- && (f instanceof Count &&
(((Count) f).isCountStar() || f.child(
- 0) instanceof Slot)
- || (f instanceof Sum && f.child(0)
instanceof Slot))
- );
- })
- .thenApply(ctx -> {
- LogicalAggregate<LogicalProject<LogicalJoin<Plan,
Plan>>> agg = ctx.root;
- return pushAgg(agg, agg.child().child(),
agg.child().getProjects());
- })
- .toRule(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN)
- );
- }
-
- private static LogicalAggregate<Plan> pushAgg(LogicalAggregate<? extends
Plan> agg,
- LogicalJoin<Plan, Plan> join, List<NamedExpression> projects) {
- List<Slot> leftOutput = join.left().getOutput();
- List<Slot> rightOutput = join.right().getOutput();
-
- List<AggregateFunction> leftAggs = new ArrayList<>();
- List<AggregateFunction> rightAggs = new ArrayList<>();
- List<Count> countStars = new ArrayList<>();
- for (AggregateFunction f : agg.getAggregateFunctions()) {
- if (f instanceof Count && ((Count) f).isCountStar()) {
- countStars.add((Count) f);
- } else {
- Slot slot = (Slot) f.child(0);
- if (leftOutput.contains(slot)) {
- leftAggs.add(f);
- } else if (rightOutput.contains(slot)) {
- rightAggs.add(f);
- } else {
- throw new IllegalStateException("Slot " + slot + " not
found in join output");
- }
- }
- }
-
- Set<Slot> leftGroupBy = new HashSet<>();
- Set<Slot> rightGroupBy = new HashSet<>();
- for (Expression e : agg.getGroupByExpressions()) {
- Slot slot = (Slot) e;
- if (leftOutput.contains(slot)) {
- leftGroupBy.add(slot);
- } else if (rightOutput.contains(slot)) {
- rightGroupBy.add(slot);
- } else {
- return null;
- }
- }
-
- if (!countStars.isEmpty() && leftGroupBy.isEmpty() &&
rightGroupBy.isEmpty()) {
- return null;
- }
-
- join.getHashJoinConjuncts().forEach(e ->
e.getInputSlots().forEach(slot -> {
- if (leftOutput.contains(slot)) {
- leftGroupBy.add(slot);
- } else if (rightOutput.contains(slot)) {
- rightGroupBy.add(slot);
- } else {
- throw new IllegalStateException("Slot " + slot + " not found
in join output");
- }
- }));
-
- Alias leftCnt = null;
- Alias rightCnt = null;
- // left agg
- Map<Slot, NamedExpression> leftSlotToOutput = new HashMap<>();
- Builder<NamedExpression> leftAggOutputBuilder =
ImmutableList.<NamedExpression>builder().addAll(leftGroupBy);
- leftAggs.forEach(func -> {
- Alias alias = func.alias(func.getName());
- leftSlotToOutput.put((Slot) func.child(0), alias);
- leftAggOutputBuilder.add(alias);
- });
- if (!rightAggs.isEmpty() || !countStars.isEmpty()) {
- leftCnt = new Count().alias("leftCntStar");
- leftAggOutputBuilder.add(leftCnt);
- }
- LogicalAggregate<Plan> leftAgg = new LogicalAggregate<>(
- ImmutableList.copyOf(leftGroupBy),
leftAggOutputBuilder.build(), join.left());
- // right agg
- Map<Slot, NamedExpression> rightSlotToOutput = new HashMap<>();
- Builder<NamedExpression> rightAggOutputBuilder =
ImmutableList.<NamedExpression>builder().addAll(rightGroupBy);
- rightAggs.forEach(func -> {
- Alias alias = func.alias(func.getName());
- rightSlotToOutput.put((Slot) func.child(0), alias);
- rightAggOutputBuilder.add(alias);
- });
- if (!leftAggs.isEmpty() || !countStars.isEmpty()) {
- rightCnt = new Count().alias("rightCntStar");
- rightAggOutputBuilder.add(rightCnt);
- }
- LogicalAggregate<Plan> rightAgg = new LogicalAggregate<>(
- ImmutableList.copyOf(rightGroupBy),
rightAggOutputBuilder.build(), join.right());
-
- Plan newJoin = join.withChildren(leftAgg, rightAgg);
-
- // top Sum agg
- // count(slot) -> sum( count(slot) * cntStar )
- // count(*) -> sum( leftCntStar * leftCntStar )
- List<NamedExpression> newOutputExprs = new ArrayList<>();
- for (NamedExpression ne : agg.getOutputExpressions()) {
- if (ne instanceof Alias && ((Alias) ne).child() instanceof
AggregateFunction) {
- AggregateFunction func = (AggregateFunction) ((Alias)
ne).child();
- if (func instanceof Count && ((Count) func).isCountStar()) {
- Preconditions.checkState(rightCnt != null && leftCnt !=
null);
- Expression multiply =
TypeCoercionUtils.processBinaryArithmetic(
- new Multiply(leftCnt.toSlot(), rightCnt.toSlot()));
- Expression expr = new Sum(multiply);
- newOutputExprs.add((NamedExpression)
ne.withChildren(expr));
- } else {
- Slot slot = (Slot) func.child(0);
- if (leftSlotToOutput.containsKey(slot)) {
- Preconditions.checkState(rightCnt != null);
- Expression multiply =
TypeCoercionUtils.processBinaryArithmetic(
- new
Multiply(leftSlotToOutput.get(slot).toSlot(), rightCnt.toSlot()));
- Expression expr = new Sum(multiply);
- newOutputExprs.add((NamedExpression)
ne.withChildren(expr));
- } else if (rightSlotToOutput.containsKey(slot)) {
- Preconditions.checkState(leftCnt != null);
- Expression multiply =
TypeCoercionUtils.processBinaryArithmetic(
- new
Multiply(rightSlotToOutput.get(slot).toSlot(), leftCnt.toSlot()));
- Expression expr = new Sum(multiply);
- newOutputExprs.add((NamedExpression)
ne.withChildren(expr));
- } else {
- throw new IllegalStateException("Slot " + slot + " not
found in join output");
- }
- }
- } else {
- newOutputExprs.add(ne);
- }
- }
- return agg.withAggOutputChild(newOutputExprs, newJoin);
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java
deleted file mode 100644
index dd09b2b52a3..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushDownAggWithDistinctThroughJoinOneSide.java
+++ /dev/null
@@ -1,176 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.rules.rewrite;
-
-import org.apache.doris.nereids.rules.Rule;
-import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.expressions.Alias;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.NamedExpression;
-import org.apache.doris.nereids.trees.expressions.Slot;
-import
org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Max;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Min;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
-import org.apache.doris.nereids.trees.plans.Plan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate;
-import org.apache.doris.nereids.trees.plans.logical.LogicalJoin;
-import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableList.Builder;
-import com.google.common.collect.Sets;
-
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * Push down agg function with distinct through join on only one side.
- */
-public class PushDownAggWithDistinctThroughJoinOneSide implements
RewriteRuleFactory {
- @Override
- public List<Rule> buildRules() {
- return ImmutableList.of(
- logicalAggregate(logicalProject(innerLogicalJoin()))
- .when(agg -> agg.child().isAllSlots())
- .when(agg ->
agg.child().child().getOtherJoinConjuncts().isEmpty())
- .when(agg -> !agg.isGenerated())
- .whenNot(agg -> agg.getAggregateFunctions().isEmpty())
- .whenNot(agg -> agg.child()
- .child(0).children().stream().anyMatch(p -> p
instanceof LogicalAggregate))
- .when(agg -> {
- Set<AggregateFunction> funcs =
agg.getAggregateFunctions();
- if (funcs.size() > 1) {
- return false;
- } else {
- return funcs.stream()
- .allMatch(f -> (f instanceof Min || f
instanceof Max || f instanceof Sum
- || f instanceof Count) &&
f.isDistinct() && f.arity() == 1
- && f.child(0) instanceof Slot);
- }
- })
- .thenApply(ctx -> {
- LogicalAggregate<LogicalProject<LogicalJoin<Plan,
Plan>>> agg = ctx.root;
- return pushDownAggWithDistinct(agg,
agg.child().child(), agg.child().getProjects());
- })
-
.toRule(RuleType.PUSH_DOWN_AGG_WITH_DISTINCT_THROUGH_JOIN_ONE_SIDE)
- );
- }
-
- private static LogicalAggregate<Plan>
pushDownAggWithDistinct(LogicalAggregate<? extends Plan> agg,
- LogicalJoin<Plan, Plan> join, List<NamedExpression> projects) {
- Plan leftJoin = join.left();
- Plan rightJoin = join.right();
- List<Slot> leftJoinOutput = leftJoin.getOutput();
- List<Slot> rightJoinOutput = rightJoin.getOutput();
-
- List<AggregateFunction> leftFuncs = new ArrayList<>();
- List<AggregateFunction> rightFuncs = new ArrayList<>();
- Set<Slot> leftFuncSlotSet = new HashSet<>();
- Set<Slot> rightFuncSlotSet = new HashSet<>();
- Set<Slot> newAggOverJoinGroupByKeys = new HashSet<>();
- for (AggregateFunction func : agg.getAggregateFunctions()) {
- Slot slot = (Slot) func.child(0);
- newAggOverJoinGroupByKeys.add(slot);
- if (leftJoinOutput.contains(slot)) {
- leftFuncs.add(func);
- leftFuncSlotSet.add(slot);
- } else if (rightJoinOutput.contains(slot)) {
- rightFuncs.add(func);
- rightFuncSlotSet.add(slot);
- } else {
- throw new IllegalStateException("Slot " + slot + " not found
in join output");
- }
- }
- boolean isLeftSideAggDistinct = !leftFuncs.isEmpty() &&
rightFuncs.isEmpty();
- boolean isRightSideAggDistinct = leftFuncs.isEmpty() &&
!rightFuncs.isEmpty();
- if (!isLeftSideAggDistinct && !isRightSideAggDistinct) {
- return null;
- }
-
- Set<Slot> leftPushDownGroupBy = new HashSet<>();
- Set<Slot> rightPushDownGroupBy = new HashSet<>();
- for (Expression e : agg.getGroupByExpressions()) {
- Slot slot = (Slot) e;
- newAggOverJoinGroupByKeys.add(slot);
- if (leftJoinOutput.contains(slot)) {
- leftPushDownGroupBy.add(slot);
- } else if (rightJoinOutput.contains(slot)) {
- rightPushDownGroupBy.add(slot);
- } else {
- return null;
- }
- }
- join.getHashJoinConjuncts().forEach(e ->
e.getInputSlots().forEach(slot -> {
- if (leftJoinOutput.contains(slot)) {
- leftPushDownGroupBy.add(slot);
- } else if (rightJoinOutput.contains(slot)) {
- rightPushDownGroupBy.add(slot);
- } else {
- throw new IllegalStateException("Slot " + slot + " not found
in join output");
- }
- }));
-
- if (isLeftSideAggDistinct) {
- leftPushDownGroupBy.add((Slot) leftFuncs.get(0).child(0));
- Builder<NamedExpression> leftAggOutputBuilder =
ImmutableList.<NamedExpression>builder()
- .addAll(leftPushDownGroupBy);
- leftJoin = new
LogicalAggregate<>(ImmutableList.copyOf(leftPushDownGroupBy),
- leftAggOutputBuilder.build(), join.left());
- } else {
- rightPushDownGroupBy.add((Slot) rightFuncs.get(0).child(0));
- Builder<NamedExpression> rightAggOutputBuilder =
ImmutableList.<NamedExpression>builder()
- .addAll(rightPushDownGroupBy);
- rightJoin = new
LogicalAggregate<>(ImmutableList.copyOf(rightPushDownGroupBy),
- rightAggOutputBuilder.build(), join.right());
- }
-
- Preconditions.checkState(leftJoin != join.left() || rightJoin !=
join.right(),
- "not pushing down aggr with distinct through join on single
side successfully");
- Plan newJoin = join.withChildren(leftJoin, rightJoin);
- LogicalAggregate<? extends Plan> newAggOverJoin =
agg.withChildGroupByAndOutput(
- ImmutableList.copyOf(newAggOverJoinGroupByKeys), projects,
newJoin);
-
- List<NamedExpression> newOutputExprs = new ArrayList<>();
- for (NamedExpression ne : agg.getOutputExpressions()) {
- if (ne instanceof Alias && ((Alias) ne).child() instanceof
AggregateFunction) {
- AggregateFunction func = (AggregateFunction) ((Alias)
ne).child();
- Slot slot = (Slot) func.child(0);
- if (leftFuncSlotSet.contains(slot) ||
rightFuncSlotSet.contains(slot)) {
- Expression newFunc = discardDistinct(func);
- newOutputExprs.add((NamedExpression)
ne.withChildren(newFunc));
- } else {
- throw new IllegalStateException("Slot " + slot + " not
found in join output");
- }
- } else {
- newOutputExprs.add(ne);
- }
- }
- return agg.withAggOutputChild(newOutputExprs, newAggOverJoin);
- }
-
- private static Expression discardDistinct(AggregateFunction func) {
- Preconditions.checkState(func.isDistinct(), "current aggregation
function is not distinct");
- Set<Expression> aggChild = Sets.newLinkedHashSet(func.children());
- return func.withDistinctAndChildren(false,
ImmutableList.copyOf(aggChild));
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
index e0219de031a..4fed1f45cb5 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriter.java
@@ -675,7 +675,7 @@ public class EagerAggRewriter extends
DefaultPlanRewriter<PushDownAggContext> {
if (context.aggFuncAndGroupKeyAllEmpty() ||
context.hasVolatileFunctions()) {
return project;
}
- if (containsVolatileGroupKeyAfterProject(project, context)) {
+ if (containsVolatileGroupKeyAfterProject(project, context) ||
project.containsNoneMovableFunction()) {
return genAggregate(project, context);
}
if (project.child() instanceof LogicalCatalogRelation
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
index 2229058b701..4dcd720ccfa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/PushDownAggregation.java
@@ -47,6 +47,7 @@ import
org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter;
import org.apache.doris.nereids.util.ExpressionUtils;
import org.apache.doris.qe.SessionVariable;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
@@ -57,6 +58,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
@@ -132,15 +134,25 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
boolean hasDecomposedAggIf = false;
boolean containsNullToNonNull = false;
Map<NamedExpression, List<AggregateFunction>>
aggFunctionsForOutputExpressions = Maps.newHashMap();
- for (NamedExpression aggOutput : agg.getOutputExpressions()) {
- List<AggregateFunction> funcs = Lists.newArrayList();
- aggFunctionsForOutputExpressions.put(aggOutput, funcs);
- for (Object obj :
aggOutput.collect(AggregateFunction.class::isInstance)) {
- AggregateFunction aggFunction = (AggregateFunction) obj;
- if (aggFunction.isDistinct()) {
- return agg;
- }
- if (pushDownAggFunctionSet.contains(aggFunction.getClass())) {
+ Set<AggregateFunction> allAggFunctions = agg.getAggregateFunctions();
+ boolean hasDistinctAgg =
allAggFunctions.stream().anyMatch(AggregateFunction::isDistinct);
+ if (hasDistinctAgg) {
+ // Keep the distinct aggregate functions unchanged at the upper
aggregate. The common distinct keys
+ // become extra group keys for duplicate elimination below joins.
+ Optional<Set<SlotReference>> distinctKeys =
getCommonDistinctKeys(allAggFunctions);
+ if (!distinctKeys.isPresent()) {
+ return agg;
+ }
+ groupKeys.addAll(distinctKeys.get());
+ } else {
+ for (NamedExpression aggOutput : agg.getOutputExpressions()) {
+ List<AggregateFunction> funcs = Lists.newArrayList();
+ aggFunctionsForOutputExpressions.put(aggOutput, funcs);
+ for (Object obj :
aggOutput.collect(AggregateFunction.class::isInstance)) {
+ AggregateFunction aggFunction = (AggregateFunction) obj;
+ if
(!pushDownAggFunctionSet.contains(aggFunction.getClass())) {
+ return agg;
+ }
if (aggFunction.containsVolatileExpression()) {
return agg;
}
@@ -187,9 +199,6 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
aggFunctions.add(aggFunction);
funcs.add(aggFunction);
}
-
- } else {
- return agg;
}
}
}
@@ -230,42 +239,47 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
// ->scan(T1[A...])
// ->scan(T2)
List<NamedExpression> newOutputExpressions = new ArrayList<>();
- for (NamedExpression ne : agg.getOutputExpressions()) {
- if (ne instanceof SlotReference) {
- newOutputExpressions.add(ne);
- } else {
- // every expression has its own replaceMap
- // aggregation(output=[min(A), sum(A)])
- // --> join
- // -> T1 [A ...]
- // -> T2 [...]
- // =>
- // aggregation(output=[min(minA), sum(sumA)])
- // --> join
- // -> agg(output=[min(A) as minA, sum(A) as
sumA])
- // -> T1 [A ...]
- // -> T2 [...]
- // for min(A), replaceMap: A->minA
- // for sum(A), replaceMap: A->sumA
- // for count(A), replaceMap: count(A)->sum(countA),
because count needs rollup to sum
- Map<Expression, Expression> replaceMap = new
HashMap<>();
- List<AggregateFunction> relatedAggFunc =
aggFunctionsForOutputExpressions.get(ne);
- for (AggregateFunction func : relatedAggFunc) {
- Alias pushedAlias =
pushDownContext.getAliasMap().get(func);
- ExprId pushId = pushedAlias.getExprId();
- if (!state.hasAggFuncOutput(pushId)) {
- continue;
- }
- Expression value =
state.getPushedAggFuncSlot(pushId);
- if (func instanceof Count) {
- replaceMap.put(func, new Sum0(value));
- } else if (func.arity() > 0) {
- replaceMap.put(func.child(0), value);
+ if (hasDistinctAgg) {
+ newOutputExpressions.addAll(agg.getOutputExpressions());
+ } else {
+ for (NamedExpression ne : agg.getOutputExpressions()) {
+ if (ne instanceof SlotReference) {
+ newOutputExpressions.add(ne);
+ } else {
+ // every expression has its own replaceMap
+ // aggregation(output=[min(A), sum(A)])
+ // --> join
+ // -> T1 [A ...]
+ // -> T2 [...]
+ // =>
+ // aggregation(output=[min(minA), sum(sumA)])
+ // --> join
+ // -> agg(output=[min(A) as minA,
sum(A) as sumA])
+ // -> T1 [A ...]
+ // -> T2 [...]
+ // for min(A), replaceMap: A->minA
+ // for sum(A), replaceMap: A->sumA
+ // for count(A), replaceMap:
count(A)->sum(countA), because count needs rollup to sum
+ Map<Expression, Expression> replaceMap = new
HashMap<>();
+ List<AggregateFunction> relatedAggFunc =
aggFunctionsForOutputExpressions.get(ne);
+ for (AggregateFunction func : relatedAggFunc) {
+ Alias pushedAlias =
pushDownContext.getAliasMap().get(func);
+ ExprId pushId = pushedAlias.getExprId();
+ if (!state.hasAggFuncOutput(pushId)) {
+ continue;
+ }
+ Expression value =
state.getPushedAggFuncSlot(pushId);
+ if (func instanceof Count) {
+ replaceMap.put(func, new Sum0(value));
+ } else if (func.arity() > 0) {
+ replaceMap.put(func.child(0), value);
+ }
}
+ NamedExpression replaceAliasExpr =
+ (NamedExpression)
ExpressionUtils.replace(ne, replaceMap);
+ replaceAliasExpr = (NamedExpression)
ExpressionUtils.rebuildSignature(replaceAliasExpr);
+ newOutputExpressions.add(replaceAliasExpr);
}
- NamedExpression replaceAliasExpr = (NamedExpression)
ExpressionUtils.replace(ne, replaceMap);
- replaceAliasExpr = (NamedExpression)
ExpressionUtils.rebuildSignature(replaceAliasExpr);
- newOutputExpressions.add(replaceAliasExpr);
}
}
LogicalAggregate<Plan> eagerAgg =
@@ -280,6 +294,33 @@ public class PushDownAggregation extends
DefaultPlanRewriter<JobContext> impleme
return agg;
}
+ private Optional<Set<SlotReference>>
getCommonDistinctKeys(Set<AggregateFunction> aggFunctions) {
+ Set<SlotReference> commonDistinctKeys = null;
+ for (AggregateFunction aggFunction : aggFunctions) {
+ if (!aggFunction.isDistinct() ||
!pushDownAggFunctionSet.contains(aggFunction.getClass())
+ || aggFunction.containsVolatileExpression()) {
+ return Optional.empty();
+ }
+ ImmutableSet.Builder<SlotReference> distinctKeysBuilder =
ImmutableSet.builder();
+ for (Expression child : aggFunction.children()) {
+ if (!(child instanceof SlotReference)) {
+ return Optional.empty();
+ }
+ distinctKeysBuilder.add((SlotReference) child);
+ }
+ Set<SlotReference> distinctKeys = distinctKeysBuilder.build();
+ if (distinctKeys.isEmpty()) {
+ return Optional.empty();
+ }
+ if (commonDistinctKeys == null) {
+ commonDistinctKeys = distinctKeys;
+ } else if (!commonDistinctKeys.equals(distinctKeys)) {
+ return Optional.empty();
+ }
+ }
+ return Optional.ofNullable(commonDistinctKeys);
+ }
+
private boolean checkSubTreePattern(Plan root) {
return containsPushDownJoin(root)
&& checkPlanNodeType(root);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHintTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHintTest.java
deleted file mode 100644
index b9e3f0142d1..00000000000
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/EliminateLogicalSelectHintTest.java
+++ /dev/null
@@ -1,56 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.rules.analysis;
-
-import org.apache.doris.nereids.hint.Hint;
-import org.apache.doris.nereids.hint.UseCboRuleHint;
-import org.apache.doris.nereids.sqltest.SqlTestBase;
-import org.apache.doris.nereids.util.PlanChecker;
-
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-import java.util.List;
-import java.util.stream.Collectors;
-
-/**
- * Tests for EliminateLogicalSelectHint, verifying that no_use_cbo_rule hints
are correctly
- * propagated into the statement context after analysis.
- */
-public class EliminateLogicalSelectHintTest extends SqlTestBase {
-
- @Test
- public void testNoUseCboRuleHintIsRecognized() {
- String sql = "SELECT /*+ no_use_cbo_rule(INFER_SET_OPERATOR_DISTINCT)
*/ * FROM T1";
- PlanChecker checker = PlanChecker.from(connectContext).analyze(sql);
-
- List<Hint> hints =
checker.getCascadesContext().getStatementContext().getHints();
- List<UseCboRuleHint> cboHints = hints.stream()
- .filter(h -> h instanceof UseCboRuleHint)
- .map(h -> (UseCboRuleHint) h)
- .collect(Collectors.toList());
-
- Assertions.assertFalse(cboHints.isEmpty(),
- "no_use_cbo_rule hint should be added to statementContext
hints");
-
Assertions.assertTrue(cboHints.stream().anyMatch(UseCboRuleHint::isNotUseCboRule),
- "hint should have isNotUseCboRule=true");
- Assertions.assertTrue(
- cboHints.stream().anyMatch(h ->
h.getHintName().equalsIgnoreCase("INFER_SET_OPERATOR_DISTINCT")),
- "hint should carry the rule name INFER_SET_OPERATOR_DISTINCT");
- }
-}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java
deleted file mode 100644
index 6f86c63aeff..00000000000
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownCountThroughJoinTest.java
+++ /dev/null
@@ -1,216 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.rules.rewrite;
-
-import org.apache.doris.common.Pair;
-import org.apache.doris.nereids.trees.expressions.Alias;
-import org.apache.doris.nereids.trees.expressions.Expression;
-import org.apache.doris.nereids.trees.expressions.Multiply;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Count;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
-import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
-import org.apache.doris.nereids.trees.plans.JoinType;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
-import org.apache.doris.nereids.types.DecimalV3Type;
-import org.apache.doris.nereids.util.LogicalPlanBuilder;
-import org.apache.doris.nereids.util.MemoPatternMatchSupported;
-import org.apache.doris.nereids.util.MemoTestUtils;
-import org.apache.doris.nereids.util.PlanChecker;
-import org.apache.doris.nereids.util.PlanConstructor;
-
-import com.google.common.collect.ImmutableList;
-import org.junit.jupiter.api.Assertions;
-import org.junit.jupiter.api.Test;
-
-class PushDownCountThroughJoinTest implements MemoPatternMatchSupported {
- private static final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
- private static final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
-
- @Test
- void testSingleCount() {
- Alias count = new Count(scan1.getOutput().get(0)).alias("count");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
ImmutableList.of(scan1.getOutput().get(0), count))
- .build();
-
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .printlnTree()
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-
- @Test
- void testMultiCount() {
- Alias leftCnt1 = new Count(scan1.getOutput().get(0)).alias("leftCnt1");
- Alias leftCnt2 = new Count(scan1.getOutput().get(1)).alias("leftCnt2");
- Alias rightCnt1 = new
Count(scan2.getOutput().get(1)).alias("rightCnt1");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
- ImmutableList.of(scan1.getOutput().get(0), leftCnt1,
leftCnt2, rightCnt1))
- .build();
-
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-
- /**
- * verify that after applying PushDownAggThroughJoin rule, agg func has
correct dataType
- */
- @Test
- void testSumAndDataType() {
- LogicalOlapScan salary1 = new
LogicalOlapScan(PlanConstructor.getNextRelationId(),
- PlanConstructor.salary, ImmutableList.of(""));
- Alias sum = new Sum(salary1.getOutput().get(2)).alias("sum");
- LogicalPlan plan = new LogicalPlanBuilder(salary1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
- ImmutableList.of(salary1.getOutput().get(0), sum))
- .build();
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- ).when(agg -> {
- Multiply multiply = null;
- for (Expression expr : agg.getOutputExpressions()) {
- if (expr instanceof Alias
- && expr.child(0) instanceof Sum
- && expr.child(0).child(0) instanceof Multiply)
{
- multiply = (Multiply) expr.child(0).child(0);
- break;
- }
- }
- if (multiply == null) {
- return false;
- }
- Assertions.assertInstanceOf(DecimalV3Type.class,
multiply.child(0).getDataType());
- Assertions.assertInstanceOf(DecimalV3Type.class,
multiply.child(1).getDataType());
- return true;
- }));
- }
-
- @Test
- void testSingleCountStar() {
- Alias count = new Count().alias("countStar");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
ImmutableList.of(scan1.getOutput().get(0), count))
- .build();
-
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-
- @Test
- void testSingleCountStarEmptyGroupBy() {
- Alias count = new Count().alias("countStar");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(),
ImmutableList.of(count))
- .build();
-
- // shouldn't rewrite.
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalOlapScan(),
- logicalOlapScan()
- )
- )
- );
- }
-
- @Test
- void testCountNullNotPushedDown() {
- // count(null) should NOT be treated as count(*) and should NOT be
pushed down.
- // NullLiteral is neither isCountStar() nor instanceof Slot,
- // so the rule's predicate rejects the aggregate.
- Alias countNull = new Count(NullLiteral.INSTANCE).alias("countNull");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
- ImmutableList.of(scan1.getOutput().get(0), countNull))
- .build();
-
- // Should NOT rewrite — aggregate stays above the original join.
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalOlapScan(),
- logicalOlapScan()
- )
- )
- );
- }
-
- @Test
- void testBothSideCountAndCountStar() {
- Alias leftCnt = new Count(scan1.getOutput().get(0)).alias("leftCnt");
- Alias rightCnt = new Count(scan2.getOutput().get(0)).alias("rightCnt");
- Alias countStar = new Count().alias("countStar");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
- ImmutableList.of(scan1.getOutput().get(0), leftCnt,
rightCnt, countStar))
- .build();
-
- PlanChecker.from(MemoTestUtils.createConnectContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java
deleted file mode 100644
index b3901c7f7b1..00000000000
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/PushDownSumThroughJoinTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.nereids.rules.rewrite;
-
-import org.apache.doris.common.Pair;
-import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.expressions.Alias;
-import org.apache.doris.nereids.trees.expressions.functions.agg.Sum;
-import org.apache.doris.nereids.trees.plans.JoinType;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
-import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
-import org.apache.doris.nereids.util.LogicalPlanBuilder;
-import org.apache.doris.nereids.util.MemoPatternMatchSupported;
-import org.apache.doris.nereids.util.MemoTestUtils;
-import org.apache.doris.nereids.util.PlanChecker;
-import org.apache.doris.nereids.util.PlanConstructor;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.SessionVariable;
-
-import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableSet;
-import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
-
-class PushDownSumThroughJoinTest implements MemoPatternMatchSupported {
- private static final LogicalOlapScan scan1 =
PlanConstructor.newLogicalOlapScan(0, "t1", 0);
- private static final LogicalOlapScan scan2 =
PlanConstructor.newLogicalOlapScan(1, "t2", 0);
-
- private ConnectContext createMockedContext() {
- ConnectContext ctx = MemoTestUtils.createConnectContext();
- SessionVariable sv = Mockito.spy(ctx.getSessionVariable());
-
Mockito.doReturn(ImmutableSet.of(RuleType.PUSH_DOWN_AGG_THROUGH_JOIN.type()))
- .when(sv).getEnableNereidsRules();
- ctx.setSessionVariable(sv);
- return ctx;
- }
-
- @Test
- void testSingleJoinLeftSum() {
- Alias sum = new Sum(scan1.getOutput().get(1)).alias("sum");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
ImmutableList.of(scan1.getOutput().get(0), sum))
- .build();
-
- PlanChecker.from(createMockedContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-
- @Test
- void testSingleJoinRightSum() {
- Alias sum = new Sum(scan2.getOutput().get(1)).alias("sum");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
ImmutableList.of(scan1.getOutput().get(0), sum))
- .build();
-
- PlanChecker.from(createMockedContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-
- @Test
- void testSingleJoinBothSum() {
- Alias leftSum = new Sum(scan1.getOutput().get(1)).alias("leftSum");
- Alias rightSum = new Sum(scan2.getOutput().get(1)).alias("rightSum");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0),
ImmutableList.of(scan1.getOutput().get(0), leftSum, rightSum))
- .build();
-
- PlanChecker.from(createMockedContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-
- @Test
- void testAggNotOutputGroupBy() {
- // agg don't output group by
- Alias sum = new Sum(scan1.getOutput().get(1)).alias("sum");
- LogicalPlan plan = new LogicalPlanBuilder(scan1)
- .join(scan2, JoinType.INNER_JOIN, Pair.of(0, 0))
- .aggGroupUsingIndex(ImmutableList.of(0), ImmutableList.of(sum))
- .build();
-
- PlanChecker.from(createMockedContext(), plan)
- .applyTopDown(new PushDownAggThroughJoin())
- .matches(
- logicalAggregate(
- logicalJoin(
- logicalAggregate(),
- logicalAggregate()
- )
- )
- );
- }
-}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
index d5eb42fa0ea..c44e76c9616 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/eageraggregation/EagerAggRewriterTest.java
@@ -19,6 +19,7 @@ package
org.apache.doris.nereids.rules.rewrite.eageraggregation;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.analyzer.UnboundRelation;
+import org.apache.doris.nereids.rules.RuleType;
import org.apache.doris.nereids.rules.exploration.join.JoinReorderContext;
import org.apache.doris.nereids.trees.expressions.Alias;
import org.apache.doris.nereids.trees.expressions.Expression;
@@ -138,19 +139,57 @@ class EagerAggRewriterTest extends TestWithFeService
implements MemoPatternMatch
}
@Test
- void testNotPushDownDistinctAgg() {
- // Distinct aggregation should not be pushed down.
+ void testPushDownDistinctAggWithSameKey() {
connectContext.getSessionVariable().setEagerAggregationMode(1);
connectContext.getSessionVariable().setDisableJoinReorder(true);
+ disableDistinctAggregateRewrite();
try {
- String sql = "select count(distinct t1.name), t2.id2 from t1 join
t2"
+ String sql = "select count(distinct t1.id1), sum(distinct t1.id1),
t2.id2 from t1 join t2"
+ " on t1.id1 = t2.id2 group by t2.id2";
- PlanChecker.from(connectContext)
- .analyze(sql)
- .rewrite()
- .nonMatch(logicalJoin(logicalAggregate(), any()))
- .nonMatch(logicalJoin(any(), logicalAggregate()))
- .printlnTree();
+ assertDistinctAggPushDown(sql, 1);
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ resetDisabledRules();
+ }
+ }
+
+ @Test
+ void testPushDownMultiColumnCountDistinct() {
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ disableDistinctAggregateRewrite();
+ try {
+ String sql = "select count(distinct t1.id1, t1.name), t2.id2 from
t1 join t2"
+ + " on t1.id1 = t2.id2 group by t2.id2";
+ assertDistinctAggPushDown(sql, 2);
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ resetDisabledRules();
+ }
+ }
+
+ @Test
+ void testNotPushDownDistinctAggWithDifferentKeys() {
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ assertNoAggregateUnderJoin("select count(distinct t1.id1),
count(distinct t1.name), t2.id2"
+ + " from t1 join t2 on t1.id1 = t2.id2 group by t2.id2");
+ } finally {
+ connectContext.getSessionVariable().setEagerAggregationMode(0);
+ connectContext.getSessionVariable().setDisableJoinReorder(false);
+ }
+ }
+
+ @Test
+ void testNotPushDownDistinctAggWithNonDistinctAgg() {
+ connectContext.getSessionVariable().setEagerAggregationMode(1);
+ connectContext.getSessionVariable().setDisableJoinReorder(true);
+ try {
+ assertNoAggregateUnderJoin("select count(distinct t1.name),
max(t1.id1), t2.id2"
+ + " from t1 join t2 on t1.id1 = t2.id2 group by t2.id2");
} finally {
connectContext.getSessionVariable().setEagerAggregationMode(0);
connectContext.getSessionVariable().setDisableJoinReorder(false);
@@ -850,6 +889,34 @@ class EagerAggRewriterTest extends TestWithFeService
implements MemoPatternMatch
Assertions.assertFalse(containsAggregateUnderJoin(plan),
plan.treeString());
}
+ private void assertDistinctAggPushDown(String sql, int
expectedLeftGroupKeyCount) {
+ Plan plan = PlanChecker.from(connectContext)
+ .analyze(sql)
+ .rewrite()
+ .getPlan();
+ LogicalAggregate<?> topAggregate = findFirstPlan(plan,
LogicalAggregate.class);
+ Assertions.assertNotNull(topAggregate, plan.treeString());
+ Assertions.assertTrue(topAggregate.getAggregateFunctions().stream()
+ .allMatch(aggFunction -> aggFunction.isDistinct()),
plan.treeString());
+ LogicalJoin<?, ?> join = findFirstPlan(topAggregate.child(),
LogicalJoin.class);
+ Assertions.assertNotNull(join, plan.treeString());
+ LogicalAggregate<?> leftAggregate = findFirstPlan(join.left(),
LogicalAggregate.class);
+ Assertions.assertNotNull(leftAggregate, plan.treeString());
+ Assertions.assertEquals(expectedLeftGroupKeyCount,
+ leftAggregate.getGroupByExpressions().size(),
plan.treeString());
+ }
+
+ private void disableDistinctAggregateRewrite() {
+
connectContext.getSessionVariable().setDisableNereidsRules(String.join(",",
+ RuleType.PRUNE_EMPTY_PARTITION.name(),
+ RuleType.DISTINCT_AGGREGATE_SPLIT.name(),
+ RuleType.PROCESS_SCALAR_AGG_MUST_USE_MULTI_DISTINCT.name()));
+ }
+
+ private void resetDisabledRules() {
+
connectContext.getSessionVariable().setDisableNereidsRules(RuleType.PRUNE_EMPTY_PARTITION.name());
+ }
+
private boolean containsAggregateUnderJoin(Plan plan) {
if (plan instanceof LogicalJoin) {
LogicalJoin<?, ?> join = (LogicalJoin<?, ?>) plan;
diff --git
a/regression-test/data/query_p0/eager_agg/distinct_agg_func_push_down.out
b/regression-test/data/query_p0/eager_agg/distinct_agg_func_push_down.out
new file mode 100644
index 00000000000..69b42b6eedc
--- /dev/null
+++ b/regression-test/data/query_p0/eager_agg/distinct_agg_func_push_down.out
@@ -0,0 +1,237 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !groupby_pushdown_basic --
+1
+1
+1
+3
+
+-- !groupby_pushdown_left_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_right_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_full_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_left_semi_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_left_anti_join --
+
+-- !groupby_pushdown_complex_conditions --
+
+-- !groupby_pushdown_with_aggregate --
+1 1
+1 2
+1 3
+3 2
+
+-- !groupby_pushdown_subquery --
+
+-- !groupby_pushdown_outer_join --
+1
+1
+1
+3
+
+-- !groupby_pushdown_deep_subquery --
+
+-- !groupby_pushdown_having --
+
+-- !groupby_pushdown_mixed_aggregates --
+1 1
+1 2
+1 3
+3 6
+
+-- !groupby_pushdown_multi_table_join --
+1
+1
+1
+
+-- !groupby_pushdown_with_order_by --
+1
+1
+1
+3
+
+-- !groupby_pushdown_multiple_equal_conditions --
+1
+1
+1
+
+-- !groupby_pushdown_equal_conditions_with_aggregate --
+1 1
+2 1
+3 1
+
+-- !groupby_pushdown_equal_conditions_non_aggregate --
+a 1
+b 1
+c 1
+
+-- !groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a 1 1
+b 1 1
+c 1 1
+
+-- !groupby_pushdown_with_where_clause --
+
+-- !groupby_pushdown_varied_aggregates --
+1 1.5 1
+1 4.5 1
+1 7.5 1
+3 7 0
+
+-- !groupby_pushdown_with_order_by_limit --
+1
+1
+1
+3
+
+-- !groupby_pushdown_alias_multiple_equal_conditions --
+1
+1
+1
+
+-- !groupby_pushdown_complex_join_condition --
+
+-- !groupby_pushdown_function_processed_columns --
+0
+1
+1
+1
+
+-- !groupby_pushdown_nested_queries --
+
+-- !with_hint_groupby_pushdown_basic --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_left_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_right_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_full_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_left_semi_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_left_anti_join --
+
+-- !with_hint_groupby_pushdown_complex_conditions --
+
+-- !with_hint_groupby_pushdown_with_aggregate --
+1 1
+1 2
+1 3
+3 2
+
+-- !with_hint_groupby_pushdown_subquery --
+
+-- !with_hint_groupby_pushdown_outer_join --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_deep_subquery --
+
+-- !with_hint_groupby_pushdown_having --
+
+-- !with_hint_groupby_pushdown_mixed_aggregates --
+1 1
+1 2
+1 3
+3 6
+
+-- !with_hint_groupby_pushdown_multi_table_join --
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_with_order_by --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_multiple_equal_conditions --
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_equal_conditions_with_aggregate --
+1 1
+2 1
+3 1
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate --
+a 1
+b 1
+c 1
+
+-- !with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate --
+a 1 1
+b 1 1
+c 1 1
+
+-- !with_hint_groupby_pushdown_with_where_clause --
+
+-- !with_hint_groupby_pushdown_varied_aggregates --
+1 1.5 1
+1 4.5 1
+1 7.5 1
+3 7 0
+
+-- !with_hint_groupby_pushdown_with_order_by_limit --
+1
+1
+1
+3
+
+-- !with_hint_groupby_pushdown_alias_multiple_equal_conditions --
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_complex_join_condition --
+
+-- !with_hint_groupby_pushdown_function_processed_columns --
+0
+1
+1
+1
+
+-- !with_hint_groupby_pushdown_nested_queries --
+
diff --git
a/regression-test/suites/query_p0/eager_agg/distinct_agg_func_push_down.groovy
b/regression-test/suites/query_p0/eager_agg/distinct_agg_func_push_down.groovy
new file mode 100644
index 00000000000..69bb4302068
--- /dev/null
+++
b/regression-test/suites/query_p0/eager_agg/distinct_agg_func_push_down.groovy
@@ -0,0 +1,256 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("push_down_count_distinct_through_join_one_side") {
+ sql "set runtime_filter_mode=OFF"
+ sql "set eager_aggregation_mode=1;"
+ sql """
+ DROP TABLE IF EXISTS count_with_distinct_t;
+ """
+
+ sql """
+ CREATE TABLE IF NOT EXISTS count_with_distinct_t(
+ `id` int(32),
+ `score` int(64) NULL,
+ `name` varchar(64) NULL
+ ) ENGINE = OLAP
+ DISTRIBUTED BY HASH(id) BUCKETS 4
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1"
+ );
+ """
+
+ sql "insert into count_with_distinct_t values (1, 1, 'a')"
+ sql "insert into count_with_distinct_t values (2, null, 'a')"
+ sql "insert into count_with_distinct_t values (3, 1, null)"
+ sql "insert into count_with_distinct_t values (4, 2, 'b')"
+ sql "insert into count_with_distinct_t values (5, null, 'b')"
+ sql "insert into count_with_distinct_t values (6, 2, null)"
+ sql "insert into count_with_distinct_t values (7, 3, 'c')"
+ sql "insert into count_with_distinct_t values (8, null, 'c')"
+ sql "insert into count_with_distinct_t values (9, 3, null)"
+ sql "insert into count_with_distinct_t values (10, null, null)"
+ sql "analyze table count_with_distinct_t with full with sync;"
+
+ order_qt_groupby_pushdown_basic """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_left_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 left
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_right_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 right
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_full_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 full
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_left_semi_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 inner
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_left_anti_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 left
anti join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_complex_conditions """
+ select count(distinct t1.score) from count_with_distinct_t t1 join
count_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by
t1.name;
+ """
+
+ order_qt_groupby_pushdown_with_aggregate """
+ select count(distinct t1.score), avg(t1.score) from
count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group
by t1.name;
+ """
+
+ order_qt_groupby_pushdown_subquery """
+ select count(distinct t1.score) from (select * from
count_with_distinct_t where score > 10) t1 join count_with_distinct_t t2 on
t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_outer_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 left
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_deep_subquery """
+ select count(distinct t1.score) from (select * from (select * from
count_with_distinct_t) count_with_distinct_t where score > 10) t1 join
count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_having """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name having
count(distinct t1.score) > 100;
+ """
+
+ order_qt_groupby_pushdown_mixed_aggregates """
+ select count(distinct t1.score), sum(distinct t1.score) from
count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group
by t1.name;
+ """
+
+ order_qt_groupby_pushdown_multi_table_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 join
count_with_distinct_t t2 on t1.id = t2.id join count_with_distinct_t t3 on
t1.name = t3.name group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_with_order_by """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+ """
+
+ order_qt_groupby_pushdown_multiple_equal_conditions """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by
t1.name;
+ """
+
+ order_qt_groupby_pushdown_equal_conditions_with_aggregate """
+ select sum(distinct t1.score), count(distinct t2.score) from
count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and
t1.name = t2.name group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_equal_conditions_non_aggregate """
+ select t1.name, count(distinct t1.score) from count_with_distinct_t
t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by
t1.name;
+ """
+
+ order_qt_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate """
+ select t1.name, count(distinct t1.score), count(distinct t2.score)
from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and
t1.name = t2.name group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_with_where_clause """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_varied_aggregates """
+ select count(distinct t1.score), avg(t1.id), count(distinct t2.name)
from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id
group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_with_order_by_limit """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by
count(distinct t1.score) limit 10;
+ """
+
+ order_qt_groupby_pushdown_alias_multiple_equal_conditions """
+ select count(distinct t1_alias.score) from count_with_distinct_t
t1_alias join count_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and
t1_alias.name = t2_alias.name group by t1_alias.name;
+ """
+
+ order_qt_groupby_pushdown_complex_join_condition """
+ select count(distinct t1.score) from count_with_distinct_t t1 join
count_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name
<> t2.name group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_function_processed_columns """
+ select count(distinct LENGTH(t1.name)) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_groupby_pushdown_nested_queries """
+ select count(distinct t1.score) from (select * from
count_with_distinct_t where score > 20) t1 join (select * from
count_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_basic """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_left_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 left
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_right_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 right
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_full_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 full
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_left_semi_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 inner
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_left_anti_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 left
anti join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_complex_conditions """
+ select count(distinct t1.score) from count_with_distinct_t t1 join
count_with_distinct_t t2 on t1.id = t2.id and t1.name < t2.name group by
t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_with_aggregate """
+ select count(distinct t1.score), avg(t1.score) from
count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group
by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_subquery """
+ select count(distinct t1.score) from (select * from
count_with_distinct_t where score > 10) t1 join count_with_distinct_t t2 on
t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_outer_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 left
join count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_deep_subquery """
+ select count(distinct t1.score) from (select * from (select * from
count_with_distinct_t) count_with_distinct_t where score > 10) t1 join
count_with_distinct_t t2 on t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_having """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name having
count(distinct t1.score) > 100;
+ """
+
+ order_qt_with_hint_groupby_pushdown_mixed_aggregates """
+ select count(distinct t1.score), sum(distinct t1.score) from
count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id group
by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_multi_table_join """
+ select count(distinct t1.score) from count_with_distinct_t t1 join
count_with_distinct_t t2 on t1.id = t2.id join count_with_distinct_t t3 on
t1.name = t3.name group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_with_order_by """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_multiple_equal_conditions """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by
t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_equal_conditions_with_aggregate """
+ select sum(distinct t1.score), count(distinct t2.score) from
count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id and
t1.name = t2.name group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate """
+ select t1.name, count(distinct t1.score) from count_with_distinct_t
t1, count_with_distinct_t t2 where t1.id = t2.id and t1.name = t2.name group by
t1.name;
+ """
+
+
order_qt_with_hint_groupby_pushdown_equal_conditions_non_aggregate_with_aggregate
"""
+ select t1.name, count(distinct t1.score), count(distinct t2.score)
from count_with_distinct_t t1, count_with_distinct_t t2 where t1.id = t2.id and
t1.name = t2.name group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_with_where_clause """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id and t1.score > 50 group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_varied_aggregates """
+ select count(distinct t1.score), avg(t1.id), count(distinct t2.name)
from count_with_distinct_t t1 join count_with_distinct_t t2 on t1.id = t2.id
group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_with_order_by_limit """
+ select count(distinct t1.score) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name order by
count(distinct t1.score) limit 10;
+ """
+
+ order_qt_with_hint_groupby_pushdown_alias_multiple_equal_conditions """
+ select count(distinct t1_alias.score) from count_with_distinct_t
t1_alias join count_with_distinct_t t2_alias on t1_alias.id = t2_alias.id and
t1_alias.name = t2_alias.name group by t1_alias.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_complex_join_condition """
+ select count(distinct t1.score) from count_with_distinct_t t1 join
count_with_distinct_t t2 on t1.id = t2.id and t1.score = t2.score and t1.name
<> t2.name group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_function_processed_columns """
+ select count(distinct LENGTH(t1.name)) from count_with_distinct_t t1,
count_with_distinct_t t2 where t1.id = t2.id group by t1.name;
+ """
+
+ order_qt_with_hint_groupby_pushdown_nested_queries """
+ select count(distinct t1.score) from (select * from
count_with_distinct_t where score > 20) t1 join (select * from
count_with_distinct_t where id < 100) t2 on t1.id = t2.id group by t1.name;
+ """
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
