This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new f20aff5dffc [improvement](statistics)Skip auto analyze empty table.
(#43865) (#44045)
f20aff5dffc is described below
commit f20aff5dffc9b741884fda14b77fb6d5699a77f3
Author: James <lijib...@selectdb.com>
AuthorDate: Sat Nov 16 23:21:43 2024 +0800
[improvement](statistics)Skip auto analyze empty table. (#43865) (#44045)
backport: https://github.com/apache/doris/pull/43865
---
.../apache/doris/statistics/AnalysisManager.java | 3 ++
.../doris/statistics/StatisticsAutoCollector.java | 15 ++++++++--
.../apache/doris/statistics/TableStatsMeta.java | 4 +++
.../statistics/StatisticsAutoCollectorTest.java | 15 ++++++++++
.../suites/statistics/test_analyze_mv.groovy | 33 ++++++++++++++++++++++
5 files changed, 68 insertions(+), 2 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
index af17a63da20..b78914442d3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -219,6 +219,9 @@ public class AnalysisManager implements Writable {
}
List<AnalysisInfo> jobs = new ArrayList<>();
autoCollector.createAnalyzeJobForTbl(stmt.getDb(), jobs,
stmt.getTable());
+ if (jobs.isEmpty()) {
+ return;
+ }
AnalysisInfo job =
autoCollector.getReAnalyzeRequiredPart(jobs.get(0));
if (job != null) {
Env.getCurrentEnv().getStatisticsAutoCollector().createSystemAnalysisJob(job);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index b4d8f1ad0ec..cf9a52da0d7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -144,7 +144,6 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
} catch (Throwable t) {
LOG.warn("Failed to analyze table {}.{}.{}",
db.getCatalog().getName(), db.getFullName(),
table.getName(), t);
- continue;
}
}
return analysisInfos;
@@ -186,7 +185,19 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
return;
}
}
- long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0
: table.getRowCount();
+ // We don't auto analyze empty table to avoid all 0 stats.
+ // Because all 0 is more dangerous than unknown stats when row count
report is delayed.
+ AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
+ TableStatsMeta tableStatsStatus =
manager.findTableStatsStatus(table.getId());
+ long rowCount = table.getRowCount();
+ if (rowCount <= 0) {
+ LOG.info("Table {} is empty, remove its old stats and skip auto
analyze it.", table.getName());
+ // Remove the table's old stats if exists.
+ if (tableStatsStatus != null &&
!tableStatsStatus.isColumnsStatsEmpty()) {
+ manager.dropStats(table);
+ }
+ return;
+ }
AnalysisInfo jobInfo = new AnalysisInfoBuilder()
.setJobId(Env.getCurrentEnv().getNextId())
.setCatalogId(db.getCatalog().getId())
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
index f39ad452e03..f24ffba5234 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/TableStatsMeta.java
@@ -242,4 +242,8 @@ public class TableStatsMeta implements Writable,
GsonPostProcessable {
protected void addIndexRowForTest(long indexId, long rowCount) {
indexesRowCount.put(indexId, rowCount);
}
+
+ public boolean isColumnsStatsEmpty() {
+ return colNameToColStatsMeta == null ||
colNameToColStatsMeta.isEmpty();
+ }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 9f2003535e1..ba50a494947 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -142,6 +142,11 @@ public class StatisticsAutoCollectorTest {
columns.add(new Column("c2", PrimitiveType.HLL));
return columns;
}
+
+ @Mock
+ public long getRowCount() {
+ return 1;
+ }
};
StatisticsAutoCollector saa = new StatisticsAutoCollector();
List<AnalysisInfo> analysisInfoList = saa.constructAnalysisInfo(new
Database(1, "anydb"));
@@ -397,6 +402,11 @@ public class StatisticsAutoCollectorTest {
objects.add(-1L);
return objects;
}
+
+ @Mock
+ public long getRowCount() {
+ return 1;
+ }
};
new MockUp<StatisticsUtil>() {
@@ -469,6 +479,11 @@ public class StatisticsAutoCollectorTest {
objects.add(-1L);
return objects;
}
+
+ @Mock
+ public long getRowCount() {
+ return 1;
+ }
};
new MockUp<StatisticsUtil>() {
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy
b/regression-test/suites/statistics/test_analyze_mv.groovy
index e3045f03018..5ae7a5ec83f 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -689,6 +689,18 @@ suite("test_analyze_mv") {
assertEquals("0", result_row[0][3])
assertEquals("-1", result_row[0][4])
+ // ** Embedded test for skip auto analyze when table is empty
+ sql """analyze table mvTestDup properties ("use.auto.analyzer" =
"true")"""
+ def empty_test = sql """show auto analyze mvTestDup"""
+ assertEquals(0, empty_test.size())
+ empty_test = sql """show column stats mvTestDup"""
+ assertEquals(0, empty_test.size())
+ // ** End of embedded test
+
+ sql """analyze table mvTestDup with sync"""
+ empty_test = sql """show column stats mvTestDup"""
+ assertEquals(12, empty_test.size())
+
for (int i = 0; i < 120; i++) {
result_row = sql """show index stats mvTestDup mv3"""
logger.info("mv3 stats: " + result_row)
@@ -703,6 +715,27 @@ suite("test_analyze_mv") {
assertEquals("mv3", result_row[0][1])
assertEquals("0", result_row[0][3])
assertEquals("0", result_row[0][4])
+
+ // ** Embedded test for skip auto analyze when table is empty again
+ sql """analyze table mvTestDup properties ("use.auto.analyzer" =
"true")"""
+ empty_test = sql """show auto analyze mvTestDup"""
+ assertEquals(0, empty_test.size())
+ empty_test = sql """show column stats mvTestDup"""
+ for (int i = 0; i < 100; i++) {
+ empty_test = sql """show column stats mvTestDup"""
+ if (empty_test.size() != 0) {
+ logger.info("async delete is not finished yet.")
+ Thread.sleep(1000)
+ }
+ break
+ }
+ assertEquals(0, empty_test.size())
+ // ** End of embedded test
+
+ sql """analyze table mvTestDup with sync"""
+ empty_test = sql """show column stats mvTestDup"""
+ assertEquals(12, empty_test.size())
+
sql """insert into mvTestDup values (1, 2, 3, 4, 5), (1, 2, 3, 4, 5),
(10, 20, 30, 40, 50), (10, 20, 30, 40, 50), (100, 200, 300, 400, 500), (1001,
2001, 3001, 4001, 5001);"""
result_row = sql """show index stats mvTestDup mv3"""
assertEquals(1, result_row.size())
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
