This is an automated email from the ASF dual-hosted git repository.
englefly pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 9a4a4c0760 [opt](Nereids)skip unknown col stats check on
__internal_scheam and information_schema (#24625)
9a4a4c0760 is described below
commit 9a4a4c0760cfd15fa4e45dbf140f326f8644b826
Author: minghong <[email protected]>
AuthorDate: Wed Sep 20 10:48:05 2023 +0800
[opt](Nereids)skip unknown col stats check on __internal_scheam and
information_schema (#24625)
columns in __internal_scheam and information_schema do not have column stats
---
.../glue/translator/PhysicalPlanTranslator.java | 7 ++++--
.../doris/nereids/stats/StatsCalculator.java | 17 +-------------
.../doris/statistics/StatisticConstants.java | 26 +++++++++++++++++++---
.../doris/statistics/StatisticsAutoCollector.java | 2 +-
.../test_forbid_unknown_col_stats.groovy | 2 ++
5 files changed, 32 insertions(+), 22 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
index 59f4318811..5db8d54d2b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java
@@ -173,6 +173,7 @@ import
org.apache.doris.planner.external.iceberg.IcebergScanNode;
import org.apache.doris.planner.external.jdbc.JdbcScanNode;
import org.apache.doris.planner.external.paimon.PaimonScanNode;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.statistics.StatisticConstants;
import org.apache.doris.tablefunction.TableValuedFunctionIf;
import org.apache.doris.thrift.TFetchOption;
import org.apache.doris.thrift.TPartitionType;
@@ -581,7 +582,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
for (int i = 0; i < slots.size(); i++) {
Slot slot = slots.get(i);
if
(olapScan.getStats().findColumnStatistics(slot).isUnKnown()
- && !isComplexDataType(slot.getDataType())) {
+ && !isComplexDataType(slot.getDataType())
+ && !StatisticConstants.isSystemTable(olapTable)) {
context.addUnknownStatsColumn(olapScanNode,
tupleDescriptor.getSlots().get(i).getId());
}
}
@@ -2043,7 +2045,8 @@ public class PhysicalPlanTranslator extends
DefaultPlanVisitor<PlanFragment, Pla
scanNode.getTupleDesc().getSlots().add(smallest);
}
try {
- if (ConnectContext.get() != null &&
ConnectContext.get().getSessionVariable().forbidUnknownColStats) {
+ if (ConnectContext.get() != null &&
ConnectContext.get().getSessionVariable().forbidUnknownColStats
+ &&
!StatisticConstants.isSystemTable(scanNode.getTupleDesc().getTable())) {
for (SlotId slotId : requiredByProjectSlotIdSet) {
if (context.isColumnStatsUnknown(scanNode, slotId)) {
throw new AnalysisException("meet unknown column stats
on table " + scanNode);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
index c3b187aad7..545d485f27 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/stats/StatsCalculator.java
@@ -18,11 +18,9 @@
package org.apache.doris.nereids.stats;
import org.apache.doris.analysis.IntLiteral;
-import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionType;
-import org.apache.doris.catalog.SchemaTable;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
@@ -627,7 +625,7 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
double rowCount = catalogRelation.getTable().estimatedRowCount();
for (SlotReference slotReference : slotSet) {
String colName = slotReference.getName();
- boolean shouldIgnoreThisCol = shouldIgnoreCol(table,
slotReference.getColumn().get());
+ boolean shouldIgnoreThisCol =
StatisticConstants.shouldIgnoreCol(table, slotReference.getColumn().get());
if (colName == null) {
throw new RuntimeException(String.format("Invalid slot: %s",
slotReference.getExprId()));
@@ -1092,17 +1090,4 @@ public class StatsCalculator extends
DefaultPlanVisitor<Statistics, Void> {
PhysicalCTEAnchor<? extends Plan, ? extends Plan> cteAnchor, Void
context) {
return groupExpression.childStatistics(1);
}
-
- private boolean shouldIgnoreCol(TableIf tableIf, Column c) {
- if (tableIf instanceof SchemaTable) {
- return true;
- }
- if (tableIf instanceof OlapTable) {
- OlapTable olapTable = (OlapTable) tableIf;
- if
(StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(olapTable.getQualifiedDbName()))
{
- return true;
- }
- }
- return !c.isVisible();
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
index c77ffa49ed..4bb7030753 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -17,6 +17,9 @@
package org.apache.doris.statistics;
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.TableIf;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.FeConstants;
import org.apache.doris.system.SystemInfoService;
@@ -59,7 +62,7 @@ public class StatisticConstants {
public static final int ANALYZE_MANAGER_INTERVAL_IN_SECS = 60;
- public static List<String> STATISTICS_DB_BLACK_LIST = new ArrayList<>();
+ public static List<String> SYSTEM_DBS = new ArrayList<>();
public static int ANALYZE_TASK_RETRY_TIMES = 5;
@@ -79,9 +82,26 @@ public class StatisticConstants {
public static final int UNION_ALL_LIMIT = 512;
static {
- STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER
+ SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER +
FeConstants.INTERNAL_DB_NAME);
- STATISTICS_DB_BLACK_LIST.add(SystemInfoService.DEFAULT_CLUSTER
+ SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
+ ClusterNamespace.CLUSTER_DELIMITER + "information_schema");
}
+
+ public static boolean isSystemTable(TableIf tableIf) {
+ if (tableIf instanceof OlapTable) {
+ OlapTable olapTable = (OlapTable) tableIf;
+ if
(StatisticConstants.SYSTEM_DBS.contains(olapTable.getQualifiedDbName())) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public static boolean shouldIgnoreCol(TableIf tableIf, Column c) {
+ if (isSystemTable(tableIf)) {
+ return true;
+ }
+ return !c.isVisible();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 64ea89ff63..3b8b7840aa 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -76,7 +76,7 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
}
Collection<DatabaseIf> dbs = ctl.getAllDbs();
for (DatabaseIf<TableIf> databaseIf : dbs) {
- if
(StatisticConstants.STATISTICS_DB_BLACK_LIST.contains(databaseIf.getFullName()))
{
+ if
(StatisticConstants.SYSTEM_DBS.contains(databaseIf.getFullName())) {
continue;
}
analyzeDb(databaseIf);
diff --git
a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
index a0d9d75499..7b4ae75bfa 100644
--- a/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
+++ b/regression-test/suites/nereids_p0/test_forbid_unknown_col_stats.groovy
@@ -49,5 +49,7 @@ suite("test_forbid_unknown_col_stats") {
exception "tables with unknown column stats: OlapScanNode{tid=0,
tblName=test_forbid_unknown_col_stats_tbl, keyRanges=, preds= limit=-1}"
}
+ sql "select count() from __internal_schema.column_statistics"
+ sql "select count() from information_schema.views"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
