This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new 1fb1f3b2119 [improvement](statistics)Skip auto analyze empty table.
(#43865) (#44099)
1fb1f3b2119 is described below
commit 1fb1f3b211927a7fdacc55c8be8b612f26fd24c9
Author: James <lijib...@selectdb.com>
AuthorDate: Mon Nov 18 13:17:22 2024 +0800
[improvement](statistics)Skip auto analyze empty table. (#43865) (#44099)
backport: https://github.com/apache/doris/pull/43865
---
.../apache/doris/statistics/AnalysisManager.java | 3 ++
.../doris/statistics/StatisticsAutoCollector.java | 15 ++++++++--
.../apache/doris/statistics/TableStatsMeta.java | 4 +++
.../statistics/StatisticsAutoCollectorTest.java | 15 ++++++++++
.../suites/statistics/test_analyze_mv.groovy | 33 ++++++++++++++++++++++
5 files changed, 68 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index a088d4fff30..5f6206e854e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -217,6 +217,9 @@ public class AnalysisManager implements Writable {
}
List<AnalysisInfo> jobs = new ArrayList<>();
autoCollector.createAnalyzeJobForTbl(stmt.getDb(), jobs,
stmt.getTable());
+ if (jobs.isEmpty()) {
+ return;
+ }
AnalysisInfo job =
autoCollector.getNeedAnalyzeColumns(jobs.get(0));
if (job != null) {
Env.getCurrentEnv().getStatisticsAutoCollector().createSystemAnalysisJob(job);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index 62d3a5b2946..c1a8af93ac0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -144,7 +144,6 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
} catch (Throwable t) {
LOG.warn("Failed to analyze table {}.{}.{}",
db.getCatalog().getName(), db.getFullName(),
table.getName(), t);
- continue;
}
}
return analysisInfos;
@@ -186,7 +185,19 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
return;
}
}
- long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0
: table.getRowCount();
+ // We don't auto analyze empty table to avoid all 0 stats.
+ // Because all 0 is more dangerous than unknown stats when row count
report is delayed.
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ TableStatsMeta tableStatsStatus =
manager.findTableStatsStatus(table.getId());
+ long rowCount = table.getRowCount();
+ if (rowCount <= 0) {
+ LOG.info("Table {} is empty, remove its old stats and skip auto
analyze it.", table.getName());
+ // Remove the table's old stats if exists.
+ if (tableStatsStatus != null &&
!tableStatsStatus.isColumnsStatsEmpty()) {
+ manager.dropStats(table);
+ }
+ return;
+ }
AnalysisInfo jobInfo = new AnalysisInfoBuilder()
.setJobId(Env.getCurrentEnv().getNextId())
.setCatalogId(db.getCatalog().getId())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index 355ae23088c..50739e98aea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -229,4 +229,8 @@ public class TableStatsMeta implements Writable,
GsonPostProcessable {
protected void addIndexRowForTest(long indexId, long rowCount) {
indexesRowCount.put(indexId, rowCount);
}
+
+ public boolean isColumnsStatsEmpty() {
+ return colToColStatsMeta == null || colToColStatsMeta.isEmpty();
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 267e04772ac..4f40071f501 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -135,6 +135,11 @@ public class StatisticsAutoCollectorTest {
columns.add(new Column("c2", PrimitiveType.HLL));
return columns;
}
+
+ @Mock
+ public long getRowCount() {
+ return 1;
+ }
};
StatisticsAutoCollector saa = new StatisticsAutoCollector();
List<AnalysisInfo> analysisInfoList = saa.constructAnalysisInfo(new
Database(1, "anydb"));
@@ -308,6 +313,11 @@ public class StatisticsAutoCollectorTest {
objects.add(-1L);
return objects;
}
+
+ @Mock
+ public long getRowCount() {
+ return 1;
+ }
};
new MockUp<StatisticsUtil>() {
@@ -376,6 +386,11 @@ public class StatisticsAutoCollectorTest {
objects.add(-1L);
return objects;
}
+
+ @Mock
+ public long getRowCount() {
+ return 1;
+ }
};
new MockUp<StatisticsUtil>() {
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy
b/regression-test/suites/statistics/test_analyze_mv.groovy
index ad3d1667595..a4263975c26 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -685,6 +685,18 @@ suite("test_analyze_mv") {
assertEquals("0", result_row[0][3])
assertEquals("-1", result_row[0][4])
+ // ** Embedded test for skip auto analyze when table is empty
+ sql """analyze table mvTestDup properties ("use.auto.analyzer" =
"true")"""
+ def empty_test = sql """show auto analyze mvTestDup"""
+ assertEquals(0, empty_test.size())
+ empty_test = sql """show column stats mvTestDup"""
+ assertEquals(0, empty_test.size())
+ // ** End of embedded test
+
+ sql """analyze table mvTestDup with sync"""
+ empty_test = sql """show column stats mvTestDup"""
+ assertEquals(12, empty_test.size())
+
for (int i = 0; i < 120; i++) {
result_row = sql """show index stats mvTestDup mv3"""
logger.info("mv3 stats: " + result_row)
@@ -699,6 +711,27 @@ suite("test_analyze_mv") {
assertEquals("mv3", result_row[0][1])
assertEquals("0", result_row[0][3])
assertEquals("0", result_row[0][4])
+
+ // ** Embedded test for skip auto analyze when table is empty again
+ sql """analyze table mvTestDup properties ("use.auto.analyzer" =
"true")"""
+ empty_test = sql """show auto analyze mvTestDup"""
+ assertEquals(0, empty_test.size())
+ empty_test = sql """show column stats mvTestDup"""
+ for (int i = 0; i < 100; i++) {
+ empty_test = sql """show column stats mvTestDup"""
+ if (empty_test.size() != 0) {
+ logger.info("async delete is not finished yet.")
+ Thread.sleep(1000)
+ }
+ break
+ }
+ assertEquals(0, empty_test.size())
+ // ** End of embedded test
+
+ sql """analyze table mvTestDup with sync"""
+ empty_test = sql """show column stats mvTestDup"""
+ assertEquals(12, empty_test.size())
+
sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5),
(10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001,
2001, 3001, 4001, 5001);"""
result_row = sql """show index stats mvTestDup mv3"""
assertEquals(1, result_row.size())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
