This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 851477e4ff5 [improvement](statistics)Skip auto analyze when table row
count is not fully reported. (return -1) (#42209) (#42405)
851477e4ff5 is described below
commit 851477e4ff5677d4b2efab8a0ef9aa247129fb71
Author: Jibing-Li <[email protected]>
AuthorDate: Thu Oct 24 18:22:46 2024 +0800
[improvement](statistics)Skip auto analyze when table row count is not
fully reported. (return -1) (#42209) (#42405)
backport: https://github.com/apache/doris/pull/42209
---
.../java/org/apache/doris/catalog/OlapTable.java | 6 ++--
.../doris/statistics/StatisticsAutoCollector.java | 7 +++++
.../statistics/StatisticsAutoCollectorTest.java | 32 ++++++++++++++++++++++
3 files changed, 43 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 5876611b4ec..3d14b92de07 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -119,6 +119,8 @@ public class OlapTable extends Table {
WAITING_STABLE
}
+ public static long ROW_COUNT_BEFORE_REPORT = -1;
+
private volatile OlapTableState state;
// index id -> index meta
@@ -1296,10 +1298,10 @@ public class OlapTable extends Table {
if (index == null) {
LOG.warn("Index {} not exist in partition {}, table {}, {}",
indexId, entry.getValue().getName(), id, name);
- return -1;
+ return ROW_COUNT_BEFORE_REPORT;
}
if (strict && !index.getRowCountReported()) {
- return -1;
+ return ROW_COUNT_BEFORE_REPORT;
}
rowCount += index.getRowCount() == -1 ? 0 : index.getRowCount();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
index a04f428aa66..137068d4315 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -179,6 +179,13 @@ public class StatisticsAutoCollector extends
StatisticsCollector {
List<AnalysisInfo> analysisInfos, TableIf table) {
AnalysisMethod analysisMethod = table.getDataSize(true) >=
StatisticsUtil.getHugeTableLowerBoundSizeInBytes()
? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
+ if (table instanceof OlapTable &&
analysisMethod.equals(AnalysisMethod.SAMPLE)) {
+ OlapTable ot = (OlapTable) table;
+ if (ot.getRowCountForIndex(ot.getBaseIndexId(), true) ==
OlapTable.ROW_COUNT_BEFORE_REPORT) {
+ LOG.info("Table {} row count is not fully reported, skip auto
analyzing this time.", ot.getName());
+ return;
+ }
+ }
long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0
: table.getRowCount();
AnalysisInfo jobInfo = new AnalysisInfoBuilder()
.setJobId(Env.getCurrentEnv().getNextId())
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
index 1ab5178c3b7..9f2003535e1 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -567,4 +567,36 @@ public class StatisticsAutoCollectorTest {
sac.analyzeAll();
}
+
+ @Test
+ public void testCreateAnalyzeJobForTbl() {
+ StatisticsAutoCollector collector = new StatisticsAutoCollector();
+ OlapTable table = new OlapTable();
+ new MockUp<OlapTable>() {
+ @Mock
+ public long getDataSize(boolean singleReplica) {
+ return 100;
+ }
+
+ @Mock
+ public long getRowCountForIndex(long indexId, boolean strict) {
+ return -1;
+ }
+
+ @Mock
+ public boolean isPartitionedTable() {
+ return false;
+ }
+ };
+ List<AnalysisInfo> infos = Lists.newArrayList();
+ collector.createAnalyzeJobForTbl(null, infos, table);
+ Assertions.assertEquals(0, infos.size());
+ new MockUp<OlapTable>() {
+ @Mock
+ public long getRowCountForIndex(long indexId, boolean strict) {
+ return 100;
+ }
+ };
+ Assertions.assertThrows(NullPointerException.class, () ->
collector.createAnalyzeJobForTbl(null, infos, table));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
