This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new c5e61289137 branch-3.0: [improve](statistics)Improve drop expired
stats function. (#54290) (#54316)
c5e61289137 is described below
commit c5e61289137589cb529dd07731a9284584a8858d
Author: James <[email protected]>
AuthorDate: Wed Aug 13 09:58:05 2025 +0800
branch-3.0: [improve](statistics)Improve drop expired stats function.
(#54290) (#54316)
backport: https://github.com/apache/doris/pull/54290
---
.../apache/doris/statistics/StatisticsCleaner.java | 13 +++--
.../doris/statistics/StatisticsCleanerTest.java | 55 ++++++++++++++++++++++
2 files changed, 64 insertions(+), 4 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
index acd90c8da14..a2fea570acb 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsCleaner.java
@@ -140,7 +140,7 @@ public class StatisticsCleaner extends MasterDaemon {
analysisManager.removeTableStats(id);
Env.getCurrentEnv().getEditLog().logDeleteTableStats(new
TableStatsDeletionLog(id));
} catch (Exception e) {
- LOG.info(e);
+ LOG.info("Fail to remove table stats for table {}", id, e);
}
}
}
@@ -256,13 +256,18 @@ public class StatisticsCleaner extends MasterDaemon {
}
}
- private long findExpiredStats(OlapTable statsTbl, ExpiredStats
expiredStats,
+ protected long findExpiredStats(OlapTable statsTbl, ExpiredStats
expiredStats,
long offset, boolean isTableColumnStats) {
long pos = offset;
- while (pos < statsTbl.getRowCount() && !expiredStats.isFull()) {
+ while (!expiredStats.isFull()) {
List<ResultRow> rows = StatisticsRepository.fetchStatsFullName(
StatisticConstants.FETCH_LIMIT, pos, isTableColumnStats);
pos += StatisticConstants.FETCH_LIMIT;
+ if (rows.isEmpty()) {
+ LOG.info("Stats table {} has no more rows to fetch.",
statsTbl.getName());
+ break;
+ }
+ LOG.info("Process {} rows in stats table {}", rows.size(),
statsTbl.getName());
for (ResultRow r : rows) {
try {
StatsId statsId = new StatsId(r);
@@ -326,7 +331,7 @@ public class StatisticsCleaner extends MasterDaemon {
return pos;
}
- private static class ExpiredStats {
+ protected static class ExpiredStats {
Set<Long> expiredCatalog = new HashSet<>();
Set<Long> expiredDatabase = new HashSet<>();
Set<Long> expiredTable = new HashSet<>();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java
new file mode 100644
index 00000000000..b06369acddf
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsCleanerTest.java
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.statistics;
+
+import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.common.Config;
+import org.apache.doris.statistics.StatisticsCleaner.ExpiredStats;
+
+import com.google.common.collect.Lists;
+import mockit.Mock;
+import mockit.MockUp;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+class StatisticsCleanerTest {
+
+ @Test
+ void testFindExpiredStats() {
+ StatisticsCleaner cleaner = new StatisticsCleaner();
+ ExpiredStats stat = new ExpiredStats();
+ OlapTable olapTable = new OlapTable();
+ for (int i = 0; i <= Config.max_allowed_in_element_num_of_delete; i++)
{
+ stat.expiredCatalog.add((long) i);
+ }
+ long expiredStats = cleaner.findExpiredStats(null, stat, 1, true);
+ Assertions.assertEquals(expiredStats, 1);
+
+ new MockUp<StatisticsRepository>() {
+ @Mock
+ public List<ResultRow> fetchStatsFullName(long limit, long offset,
boolean isTableStats) {
+ return Lists.newArrayList();
+ }
+ };
+ stat.expiredCatalog.clear();
+ expiredStats = cleaner.findExpiredStats(olapTable, stat, 0, true);
+ Assertions.assertEquals(expiredStats, StatisticConstants.FETCH_LIMIT);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
