This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 61f33243d5 [improvement](nereids) Add logs and config options (#22580)
61f33243d5 is described below
commit 61f33243d59fae228862da387e950f3d81770e08
Author: AKIRA <33112463+kikyou1...@users.noreply.github.com>
AuthorDate: Thu Aug 3 23:23:52 2023 +0800
[improvement](nereids) Add logs and config options (#22580)
---
.../org/apache/doris/analysis/ShowAnalyzeStmt.java | 2 +-
.../doris/nereids/stats/StatsCalculator.java | 24 +++++++++++++---------
.../java/org/apache/doris/qe/SessionVariable.java | 5 +++++
.../apache/doris/statistics/BaseAnalysisTask.java | 2 ++
.../apache/doris/statistics/OlapAnalysisTask.java | 10 ++++-----
.../apache/doris/statistics/StatisticsCache.java | 4 +---
.../doris/statistics/util/StatisticsUtil.java | 7 +++++++
.../nereids/jobs/cascades/DeriveStatsJobTest.java | 6 +++++-
.../doris/nereids/stats/StatsCalculatorTest.java | 9 ++++++--
9 files changed, 47 insertions(+), 22 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
index 3a0aa05a7d..95035641a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowAnalyzeStmt.java
@@ -244,7 +244,7 @@ public class ShowAnalyzeStmt extends ShowStmt {
if (!valid) {
throw new AnalysisException("Where clause should looks like: "
- + "STATE =
\"PENDING|SCHEDULING|RUNNING|FINISHED|FAILED|CANCELLED\"");
+ + "STATE = \"PENDING|RUNNING|FINISHED|FAILED");
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index 472e430693..832914360b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -562,13 +562,14 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
}
private Histogram getColumnHistogram(TableIf table, String colName) {
- if (totalHistogramMap.get(table.getName() + colName) != null) {
- return totalHistogramMap.get(table.getName() + colName);
- } else if (isPlayNereidsDump) {
- return null;
- } else {
- return
Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName);
- }
+ // if (totalHistogramMap.get(table.getName() + colName) != null) {
+ // return totalHistogramMap.get(table.getName() + colName);
+ // } else if (isPlayNereidsDump) {
+ // return null;
+ // } else {
+ // return
Env.getCurrentEnv().getStatisticsCache().getHistogram(table.getId(), colName);
+ // }
+ return null;
}
// TODO: 1. Subtract the pruned partition
@@ -582,18 +583,21 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
double rowCount = catalogRelation.getTable().estimatedRowCount();
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getName();
+ boolean shouldIgnoreThisCol = shouldIgnoreCol(table,
slotReference.getColumn().get());
+
if (colName == null) {
throw new RuntimeException(String.format("Invalid slot: %s",
slotReference.getExprId()));
}
ColumnStatistic cache = Config.enable_stats &&
FeConstants.enableInternalSchemaDb
- ? getColumnStatistic(table, colName) :
ColumnStatistic.UNKNOWN;
+ ? shouldIgnoreThisCol
+ ? ColumnStatistic.UNKNOWN : getColumnStatistic(table,
colName) : ColumnStatistic.UNKNOWN;
if (cache.avgSizeByte <= 0) {
cache = new ColumnStatisticBuilder(cache)
.setAvgSizeByte(slotReference.getColumn().get().getType().getSlotSize())
.build();
}
if (cache.isUnKnown) {
- if (forbidUnknownColStats &&
!ignoreUnknownColStatsCheck(table, slotReference.getColumn().get())) {
+ if (forbidUnknownColStats && !shouldIgnoreThisCol) {
if (StatisticsUtil.statsTblAvailable()) {
throw new AnalysisException(String.format("Found
unknown stats for column:%s.%s.\n"
+ "It may caused by:\n"
@@ -1064,7 +1068,7 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
return groupExpression.childStatistics(1);
}
- private boolean ignoreUnknownColStatsCheck(TableIf tableIf, Column c) {
+ private boolean shouldIgnoreCol(TableIf tableIf, Column c) {
if (tableIf instanceof SchemaTable) {
return true;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index d10e4a81a6..12f8dc1bc8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -359,6 +359,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_STRONG_CONSISTENCY =
"enable_strong_consistency_read";
+ public static final String PARALLEL_SYNC_ANALYZE_TASK_NUM =
"parallel_sync_analyze_task_num";
+
public static final String CBO_CPU_WEIGHT = "cbo_cpu_weight";
public static final String CBO_MEM_WEIGHT = "cbo_mem_weight";
@@ -1014,6 +1016,9 @@ public class SessionVariable implements Serializable,
Writable {
})
public boolean enableStrongConsistencyRead = false;
+ @VariableMgr.VarAttr(name = PARALLEL_SYNC_ANALYZE_TASK_NUM)
+ public int parallelSyncAnalyzeTaskNum = 2;
+
// If this fe is in fuzzy mode, then will use initFuzzyModeVariables to
generate some variables,
// not the default value set in the code.
public void initFuzzyModeVariables() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
index 9c15c49c38..835691fdc5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/BaseAnalysisTask.java
@@ -35,6 +35,7 @@ import org.apache.logging.log4j.Logger;
import java.util.HashSet;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
public abstract class BaseAnalysisTask {
@@ -174,6 +175,7 @@ public abstract class BaseAnalysisTask {
if (retriedTimes >
StatisticConstants.ANALYZE_TASK_RETRY_TIMES) {
throw new RuntimeException(t);
}
+ StatisticsUtil.sleep(TimeUnit.SECONDS.toMillis(2 ^
retriedTimes) * 10);
}
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
index 9fb04f9394..04aab26831 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/OlapAnalysisTask.java
@@ -50,11 +50,6 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
+ " (SELECT NDV(`${colName}`) AS ndv "
+ " FROM `${dbName}`.`${tblName}` ${sampleExpr}) t2\n";
- @VisibleForTesting
- public OlapAnalysisTask() {
- super();
- }
-
public OlapAnalysisTask(AnalysisInfo info) {
super(info);
}
@@ -113,6 +108,8 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
if (killed) {
return;
}
+ long startTime = System.currentTimeMillis();
+ LOG.info("ANALYZE SQL : " + sql + " start at " + startTime);
try (AutoCloseConnectContext r = StatisticsUtil.buildConnectContext())
{
r.connectContext.getSessionVariable().disableNereidsPlannerOnce();
stmtExecutor = new StmtExecutor(r.connectContext, sql);
@@ -123,6 +120,9 @@ public class OlapAnalysisTask extends BaseAnalysisTask {
throw new RuntimeException(String.format("Failed to analyze
%s.%s.%s, error: %s sql: %s",
info.catalogName, info.dbName, info.colName, sql,
queryState.getErrorMessage()));
}
+ } finally {
+ LOG.info("Analyze SQL: " + sql + " cost time: " +
(System.currentTimeMillis() - startTime) + "ms");
}
}
+
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
index 1149ecdd5a..33375a9c25 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCache.java
@@ -64,7 +64,6 @@ public class StatisticsCache {
private final AsyncLoadingCache<StatisticsCacheKey,
Optional<ColumnStatistic>> columnStatisticsCache =
Caffeine.newBuilder()
.maximumSize(Config.stats_cache_size)
-
.expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS))
.refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
.executor(threadPool)
.buildAsync(columnStatisticsCacheLoader);
@@ -72,7 +71,6 @@ public class StatisticsCache {
private final AsyncLoadingCache<StatisticsCacheKey, Optional<Histogram>>
histogramCache =
Caffeine.newBuilder()
.maximumSize(Config.stats_cache_size)
-
.expireAfterAccess(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_VALID_DURATION_IN_HOURS))
.refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
.executor(threadPool)
.buildAsync(histogramCacheLoader);
@@ -80,7 +78,7 @@ public class StatisticsCache {
private final AsyncLoadingCache<StatisticsCacheKey,
Optional<TableStatistic>> tableStatisticsCache =
Caffeine.newBuilder()
.maximumSize(Config.stats_cache_size)
-
.expireAfterWrite(Duration.ofHours(StatisticConstants.ROW_COUNT_CACHE_VALID_DURATION_IN_HOURS))
+
.refreshAfterWrite(Duration.ofHours(StatisticConstants.STATISTICS_CACHE_REFRESH_INTERVAL))
.executor(threadPool)
.buildAsync(tableStatisticsCacheLoader);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
index 61551b4f20..b2e4baecee 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/util/StatisticsUtil.java
@@ -664,4 +664,11 @@ public class StatisticsUtil {
|| type instanceof VariantType;
}
+ public static void sleep(long millis) {
+ try {
+ Thread.sleep(millis);
+ } catch (InterruptedException ignore) {
+ // IGNORE
+ }
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
index c6c60c815b..fd60eb2e00 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/jobs/cascades/DeriveStatsJobTest.java
@@ -17,11 +17,14 @@
package org.apache.doris.nereids.jobs.cascades;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.nereids.CascadesContext;
import org.apache.doris.nereids.jobs.JobContext;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.Alias;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.SlotReference;
import org.apache.doris.nereids.trees.expressions.StatementScopeIdGenerator;
@@ -74,7 +77,8 @@ public class DeriveStatsJobTest {
long tableId1 = 0;
List<String> qualifier = ImmutableList.of("test", "t");
- slot1 = new SlotReference("c1", IntegerType.INSTANCE, true, qualifier);
+ slot1 = new SlotReference(new ExprId(1), "c1", IntegerType.INSTANCE,
true, qualifier,
+ new Column("e", PrimitiveType.INT));
new Expectations() {{
ConnectContext.get();
result = context;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
index 4ca19dedf7..0fa8a17863 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/nereids/stats/StatsCalculatorTest.java
@@ -17,12 +17,15 @@
package org.apache.doris.nereids.stats;
+import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.nereids.memo.Group;
import org.apache.doris.nereids.memo.GroupExpression;
import org.apache.doris.nereids.properties.LogicalProperties;
import org.apache.doris.nereids.trees.expressions.And;
import org.apache.doris.nereids.trees.expressions.EqualTo;
+import org.apache.doris.nereids.trees.expressions.ExprId;
import org.apache.doris.nereids.trees.expressions.Expression;
import org.apache.doris.nereids.trees.expressions.Or;
import org.apache.doris.nereids.trees.expressions.SlotReference;
@@ -249,7 +252,8 @@ public class StatsCalculatorTest {
public void testOlapScan(@Mocked ConnectContext context) {
long tableId1 = 0;
List<String> qualifier = ImmutableList.of("test", "t");
- SlotReference slot1 = new SlotReference("c1", IntegerType.INSTANCE,
true, qualifier);
+ SlotReference slot1 = new SlotReference(new ExprId(0),
+ "c1", IntegerType.INSTANCE, true, qualifier, new Column("c1",
PrimitiveType.INT));
OlapTable table1 = PlanConstructor.newOlapTable(tableId1, "t1", 0);
LogicalOlapScan logicalOlapScan1 = (LogicalOlapScan) new
LogicalOlapScan(StatementScopeIdGenerator.newRelationId(), table1,
@@ -270,7 +274,8 @@ public class StatsCalculatorTest {
List<String> qualifier = new ArrayList<>();
qualifier.add("test");
qualifier.add("t");
- SlotReference slot1 = new SlotReference("c1", IntegerType.INSTANCE,
true, qualifier);
+ SlotReference slot1 = new SlotReference(new ExprId(0),
+ "c1", IntegerType.INSTANCE, true, qualifier, new Column("c1",
PrimitiveType.INT));
ColumnStatisticBuilder columnStat1 = new ColumnStatisticBuilder();
columnStat1.setNdv(10);
columnStat1.setNumNulls(5);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
