This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 668a68967ca0de175bb5840778d250b0a6116dc2
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Fri Jan 19 10:47:15 2024 +0800
[fix](statistics)Reanalyze olapTable if getRowCount is not 0 and last time
row count is 0 (#30096)
Sample analyze may write 0 result if getRowCount is not updated while
analyzing. So we need to reanalyze the table if getRowCount > 0 and previous
analyze row count is 0. Otherwise the stats for this table may stay 0 for ever
before user load new data to this table.
---
fe/fe-common/src/main/java/org/apache/doris/common/Config.java | 2 +-
fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java | 3 +++
.../src/main/java/org/apache/doris/statistics/AnalysisManager.java | 2 +-
.../java/org/apache/doris/statistics/StatisticsAutoCollector.java | 2 +-
.../src/main/java/org/apache/doris/statistics/TableStatsMeta.java | 2 +-
.../test/java/org/apache/doris/statistics/AnalysisManagerTest.java | 7 ++++++-
6 files changed, 13 insertions(+), 5 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 01fd3596895..00d3ebb8a9b 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1565,7 +1565,7 @@ public class Config extends ConfigBase {
"This parameter controls the time interval for automatic
collection jobs to check the health of table"
+ "statistics and trigger automatic collection"
})
- public static int auto_check_statistics_in_minutes = 10;
+ public static int auto_check_statistics_in_minutes = 5;
/**
* If set to TRUE, the compaction slower replica will be skipped when
select get queryable replicas
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 3236fc09ec9..b9479aeddfa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1193,6 +1193,9 @@ public class OlapTable extends Table {
return true;
}
long rowCount = getRowCount();
+ if (rowCount > 0 && tblStats.rowCount == 0) {
+ return true;
+ }
long updateRows = tblStats.updatedRows.get();
int tblHealth = StatisticsUtil.getTableHealth(rowCount, updateRows);
return tblHealth < StatisticsUtil.getTableStatsHealthThreshold();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index 2e12b443390..8f927694dc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -545,7 +545,7 @@ public class AnalysisManager implements Writable {
}
TableStatsMeta tableStats = findTableStatsStatus(tbl.getId());
if (tableStats == null) {
- updateTableStatsStatus(new TableStatsMeta(tbl.estimatedRowCount(),
jobInfo, tbl));
+ updateTableStatsStatus(new TableStatsMeta(jobInfo.emptyJob ? 0 :
tbl.estimatedRowCount(), jobInfo, tbl));
} else {
tableStats.update(jobInfo, tbl);
logCreateTableStats(tableStats);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index e2eeb21aad4..a09b56acfcc 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -209,7 +209,7 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
@VisibleForTesting
protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
TableIf table = StatisticsUtil.findTable(jobInfo.catalogId,
jobInfo.dbId, jobInfo.tblId);
- // Skip tables that are too width.
+ // Skip tables that are too wide.
if (table.getBaseSchema().size() >
StatisticsUtil.getAutoAnalyzeTableWidthThreshold()) {
return null;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index 926194a7258..9231c6a2bc7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -157,7 +157,7 @@ public class TableStatsMeta implements Writable {
jobType = analyzedJob.jobType;
if (tableIf != null) {
if (tableIf instanceof OlapTable) {
- rowCount = tableIf.getRowCount();
+ rowCount = analyzedJob.emptyJob ? 0 : tableIf.getRowCount();
}
if (!analyzedJob.emptyJob && analyzedJob.colToPartitions.keySet()
.containsAll(tableIf.getBaseSchema().stream()
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
index 2fc6d24e305..f8a77fe06db 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/AnalysisManagerTest.java
@@ -275,7 +275,7 @@ public class AnalysisManagerTest {
new MockUp<OlapTable>() {
int count = 0;
- int[] rowCount = new int[]{100, 100, 200, 200};
+ int[] rowCount = new int[]{100, 100, 200, 200, 1, 1};
final Column c = new Column("col1", PrimitiveType.INT);
@Mock
@@ -304,6 +304,11 @@ public class AnalysisManagerTest {
.setColToPartitions(new
HashMap<>()).setColName("col1").build(), olapTable);
stats2.updatedRows.addAndGet(20);
Assertions.assertFalse(olapTable.needReAnalyzeTable(stats2));
+
+ TableStatsMeta stats3 = new TableStatsMeta(0, new AnalysisInfoBuilder()
+ .setColToPartitions(new
HashMap<>()).setEmptyJob(true).setColName("col1").build(), olapTable);
+ Assertions.assertTrue(olapTable.needReAnalyzeTable(stats3));
+
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
