This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a4a3e7e8321 [feat](nereids)disable join reorder if any table row count
is not available #43000 (branch-2.0） (#43245)
a4a3e7e8321 is described below
commit a4a3e7e832100e32fe0a86e26eca5bda3f8c76da
Author: minghong <engle...@gmail.com>
AuthorDate: Tue Nov 5 18:29:28 2024 +0800
[feat](nereids)disable join reorder if any table row count is not available
#43000 (branch-2.0） (#43245)
### What problem does this PR solve?
pick #43000
<!--
You need to clearly describe your PR in this part:
1. What problem was fixed (it's best to include specific error reporting
information). How it was fixed.
2. Which behaviors were modified. What was the previous behavior, what
is it now, why was it modified, and what possible impacts might there
be.
3. What features were added. Why this function was added.
4. Which codes were refactored and why this part of the code was
refactored.
5. Which functions were optimized and what is the difference before and
after the optimization.
The description of the PR needs to enable reviewers to quickly and
clearly understand the logic of the code modification.
-->
<!--
If there are related issues, please fill in the issue number.
- If you want the issue to be closed after the PR is merged, please use
"close #12345". Otherwise, use "ref #12345"
-->
Issue Number: close #xxx
<!--
If this PR is followup a preivous PR, for example, fix the bug that
introduced by a related PR,
link the PR here
-->
Related PR: #xxx
Problem Summary:
### Check List (For Committer)
- Test <!-- At least one of them must be included. -->
- [ ] Regression test
- [ ] Unit Test
- [ ] Manual test (add detailed scripts or steps below)
- [ ] No need to test or manual test. Explain why:
- [ ] This is a refactor/code format and no logic has been changed.
- [ ] Previous test can cover this change.
- [ ] No colde files have been changed.
- [ ] Other reason <!-- Add your reason? -->
- Behavior changed:
- [ ] No.
- [ ] Yes. <!-- Explain the behavior change -->
- Does this need documentation?
- [ ] No.
- [ ] Yes. <!-- Add document PR link here. eg:
https://github.com/apache/doris-website/pull/1214 -->
- Release note
<!-- bugfix, feat, behavior changed need a release note -->
<!-- Add one line release note for this PR. -->
None
### Check List (For Reviewer who merge this PR)
- [ ] Confirm the release note
- [ ] Confirm test cases
- [ ] Confirm document
- [ ] Add branch pick label <!-- Add branch pick label that this PR
should merge into -->
---
.../org/apache/doris/nereids/NereidsPlanner.java | 6 +--
.../doris/nereids/stats/StatsCalculator.java | 48 +++++++++++-----------
2 files changed, 26 insertions(+), 28 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
index 24550f4f724..85074d2ab9e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/NereidsPlanner.java
@@ -50,8 +50,8 @@ import
org.apache.doris.nereids.trees.expressions.NamedExpression;
import org.apache.doris.nereids.trees.expressions.literal.Literal;
import org.apache.doris.nereids.trees.plans.AbstractPlan;
import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.algebra.CatalogRelation;
import
org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
import org.apache.doris.nereids.trees.plans.physical.PhysicalOneRowRelation;
import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan;
@@ -266,8 +266,8 @@ public class NereidsPlanner extends Planner {
// 1. user set leading hint
// 2. ut test. In ut test, FeConstants.enableInternalSchemaDb is
false or FeConstants.runningUnitTest is true
if (FeConstants.enableInternalSchemaDb && !FeConstants.runningUnitTest
&& cascadesContext.isLeadingJoin()) {
- List<LogicalOlapScan> scans = cascadesContext.getRewritePlan()
- .collectToList(LogicalOlapScan.class::isInstance);
+ List<CatalogRelation> scans = cascadesContext.getRewritePlan()
+ .collectToList(CatalogRelation.class::isInstance);
Optional<String> reason =
StatsCalculator.disableJoinReorderIfStatsInvalid(scans, cascadesContext);
reason.ifPresent(LOG::info);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 1fc3708545e..b088ed0dccf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -655,10 +655,6 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
idxId = olapScan.getSelectedIndexId();
}
}
- // if (deltaRowCount > 0 && LOG.isDebugEnabled()) {
- // LOG.debug("{} is partially analyzed, clear min/max values in
column stats",
- // catalogRelation.getTable().getName());
- // }
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getColumn().isPresent()
? slotReference.getColumn().get().getName()
@@ -692,14 +688,6 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
hasUnknownCol = true;
}
if (ConnectContext.get() != null &&
ConnectContext.get().getSessionVariable().enableStats) {
- // if (deltaRowCount > 0) {
- // // clear min-max to avoid error estimation
- // // for example, after yesterday data loaded, user send
query about yesterday immediately.
- // // since yesterday data are not analyzed, the max date
is before yesterday, and hence optimizer
- // // estimates the filter result is zero
- //
colStatsBuilder.setMinExpr(null).setMinValue(Double.NEGATIVE_INFINITY)
- //
.setMaxExpr(null).setMaxValue(Double.POSITIVE_INFINITY);
- // }
columnStatisticBuilderMap.put(slotReference, colStatsBuilder);
} else {
columnStatisticBuilderMap.put(slotReference, new
ColumnStatisticBuilder(ColumnStatistic.UNKNOWN));
@@ -1201,34 +1189,44 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
* 2. col stats ndv=0 but minExpr or maxExpr is not null
* 3. ndv > 10 * rowCount
*/
- public static Optional<String>
disableJoinReorderIfStatsInvalid(List<LogicalOlapScan> scans,
+ public static Optional<String>
disableJoinReorderIfStatsInvalid(List<CatalogRelation> scans,
CascadesContext context) {
StatsCalculator calculator = new StatsCalculator(context);
if (ConnectContext.get() == null) {
// ut case
return Optional.empty();
}
- for (LogicalOlapScan scan : scans) {
- double rowCount = calculator.getOlapTableRowCount(scan);
+ for (CatalogRelation scan : scans) {
+ double rowCount = calculator.getTableRowCount(scan);
// row count not available
if (rowCount == -1) {
LOG.info("disable join reorder since row count not available: "
+ scan.getTable().getNameWithFullQualifiers());
return Optional.of("table[" + scan.getTable().getName() + "]
row count is invalid");
}
- // ndv abnormal
- Optional<String> reason = calculator.checkNdvValidation(scan,
rowCount);
- if (reason.isPresent()) {
- try {
-
ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce();
- LOG.info("disable join reorder since col stats invalid: "
- + reason.get());
- } catch (Exception e) {
- LOG.info("disableNereidsJoinReorderOnce failed");
+ if (scan instanceof OlapScan) {
+ // ndv abnormal
+ Optional<String> reason =
calculator.checkNdvValidation((OlapScan) scan, rowCount);
+ if (reason.isPresent()) {
+ try {
+
ConnectContext.get().getSessionVariable().disableNereidsJoinReorderOnce();
+ LOG.info("disable join reorder since col stats
invalid: "
+ + reason.get());
+ } catch (Exception e) {
+ LOG.info("disableNereidsJoinReorderOnce failed");
+ }
+ return reason;
}
- return reason;
}
}
return Optional.empty();
}
+
+ private double getTableRowCount(CatalogRelation scan) {
+ if (scan instanceof OlapScan) {
+ return getOlapTableRowCount((OlapScan) scan);
+ } else {
+ return scan.getTable().getRowCount();
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
