This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 9f16fba8cfd branch-3.1: [fix](auto bucket)Fix auto bucket calc
bucketnum err when partition size is invalid #52801 (#53249)
9f16fba8cfd is described below
commit 9f16fba8cfde6ccfbeacaad95620bca71151d777
Author: deardeng <[email protected]>
AuthorDate: Wed Jul 16 11:00:24 2025 +0800
branch-3.1: [fix](auto bucket)Fix auto bucket calc bucketnum err when
partition size is invalid #52801 (#53249)
cherry pick from #52801
---
.../main/java/org/apache/doris/common/Config.java | 6 +
.../org/apache/doris/analysis/ShowDataStmt.java | 2 +-
.../apache/doris/catalog/MaterializedIndex.java | 4 +-
.../org/apache/doris/catalog/MetadataViewer.java | 2 +-
.../java/org/apache/doris/catalog/OlapTable.java | 2 +-
.../java/org/apache/doris/catalog/Partition.java | 14 ++-
.../main/java/org/apache/doris/catalog/Tablet.java | 5 +-
.../clone/ColocateTableCheckerAndBalancer.java | 3 +-
.../doris/clone/DynamicPartitionScheduler.java | 136 ++++++++++++++++-----
.../apache/doris/cloud/CacheHotspotManager.java | 6 +-
.../doris/common/proc/TabletHealthProcDir.java | 2 +-
.../org/apache/doris/catalog/CatalogTestUtil.java | 2 +-
.../doris/catalog/DynamicPartitionTableTest.java | 46 ++++++-
.../doris/common/util/AutoBucketUtilsTest.java | 1 +
14 files changed, 182 insertions(+), 49 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index ef8cebdb0ad..ab09c68998f 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -2706,6 +2706,12 @@ public class Config extends ConfigBase {
})
public static int autobucket_partition_size_per_bucket_gb = -1;
+ @ConfField(mutable = true, masterOnly = true, description = {"Auto
bucket中计算出的新的分区bucket num超过前一个分区的"
+ + "bucket num的百分比，被认为是异常case报警",
+ "The new partition bucket number calculated in the auto bucket
exceeds the percentage "
+ + "of the previous partition's bucket number, which is considered
an abnormal case alert."})
+ public static double autobucket_out_of_bounds_percent_threshold = 0.5;
+
@ConfField(description = {"(已弃用，被 arrow_flight_max_connection 替代) Arrow
Flight Server中所有用户token的缓存上限，"
+ "超过后LRU淘汰, arrow flight sql是无状态的协议，连接通常不会主动断开，"
+ "bearer token 从 cache 淘汰的同时会 unregister Connection.",
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
index c7bee06ffda..bc2ac9d5d7c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
@@ -298,7 +298,7 @@ public class ShowDataStmt extends ShowStmt implements
NotFallbackInParser {
long remoteSegmentSize = 0;
for (Partition partition : table.getAllPartitions()) {
MaterializedIndex mIndex = partition.getIndex(indexId);
- indexSize += mIndex.getDataSize(false);
+ indexSize += mIndex.getDataSize(false, false);
indexReplicaCount += mIndex.getReplicaCount();
indexRowCount += mIndex.getRowCount() == -1 ? 0 :
mIndex.getRowCount();
indexRemoteSize += mIndex.getRemoteDataSize();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
index 5e367b538cd..b5a31a32e1e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MaterializedIndex.java
@@ -173,10 +173,10 @@ public class MaterializedIndex extends MetaObject
implements GsonPostProcessable
this.rollupFinishedVersion = -1L;
}
- public long getDataSize(boolean singleReplica) {
+ public long getDataSize(boolean singleReplica, boolean filterSizeZero) {
long dataSize = 0;
for (Tablet tablet : getTablets()) {
- dataSize += tablet.getDataSize(singleReplica);
+ dataSize += tablet.getDataSize(singleReplica, filterSizeZero);
}
return dataSize;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
index 66cdfa5b42d..e44b7ec52c8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MetadataViewer.java
@@ -332,7 +332,7 @@ public class MetadataViewer {
for (int i = 0; i < tabletIds.size(); i++) {
Tablet tablet = mIndex.getTablet(tabletIds.get(i));
long rowCount = tablet.getRowCount(true);
- long dataSize = tablet.getDataSize(true);
+ long dataSize = tablet.getDataSize(true, false);
rowCountTabletInfos.set(i, rowCountTabletInfos.get(i)
+ rowCount);
dataSizeTabletInfos.set(i, dataSizeTabletInfos.get(i)
+ dataSize);
totalSize += dataSize;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index b94ccbb42d2..5248744dd98 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1594,7 +1594,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
long dataSize = 0;
for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
rowCount += entry.getValue().getBaseIndex().getRowCount();
- dataSize += entry.getValue().getBaseIndex().getDataSize(false);
+ dataSize += entry.getValue().getBaseIndex().getDataSize(false,
false);
}
if (rowCount > 0) {
return dataSize / rowCount;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
index 0468d3a7549..e00bf4b28ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Partition.java
@@ -291,7 +291,7 @@ public class Partition extends MetaObject {
public long getDataSize(boolean singleReplica) {
long dataSize = 0;
for (MaterializedIndex mIndex :
getMaterializedIndices(IndexExtState.VISIBLE)) {
- dataSize += mIndex.getDataSize(singleReplica);
+ dataSize += mIndex.getDataSize(singleReplica, false);
}
return dataSize;
}
@@ -458,7 +458,7 @@ public class Partition extends MetaObject {
public long getAvgRowLength() {
long rowCount = getBaseIndex().getRowCount();
- long dataSize = getBaseIndex().getDataSize(false);
+ long dataSize = getBaseIndex().getDataSize(false, false);
if (rowCount > 0) {
return dataSize / rowCount;
} else {
@@ -467,6 +467,14 @@ public class Partition extends MetaObject {
}
public long getDataLength() {
- return getBaseIndex().getDataSize(false);
+ return getBaseIndex().getDataSize(false, false);
+ }
+
+ public long getDataSizeExcludeEmptyReplica(boolean singleReplica) {
+ long dataSize = 0;
+ for (MaterializedIndex mIndex :
getMaterializedIndices(IndexExtState.VISIBLE)) {
+ dataSize += mIndex.getDataSize(singleReplica, true);
+ }
+ return dataSize + getRemoteDataSize();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
index ea76273e6df..f18460a320d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Tablet.java
@@ -515,8 +515,11 @@ public class Tablet extends MetaObject {
return id == tablet.id;
}
- public long getDataSize(boolean singleReplica) {
+ // ATTN: Replica::getDataSize may zero in cloud and non-cloud
+ // due to dataSize not write to image
+ public long getDataSize(boolean singleReplica, boolean filterSizeZero) {
LongStream s = replicas.stream().filter(r -> r.getState() ==
ReplicaState.NORMAL)
+ .filter(r -> !filterSizeZero || r.getDataSize() > 0)
.mapToLong(Replica::getDataSize);
return singleReplica ?
Double.valueOf(s.average().orElse(0)).longValue() : s.sum();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
index 7727bc77e18..7dc0a9125dd 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/ColocateTableCheckerAndBalancer.java
@@ -655,7 +655,8 @@ public class ColocateTableCheckerAndBalancer extends
MasterDaemon {
bucketsSeq.size() + " vs. " +
replicationNum);
Tablet tablet = index.getTablet(tabletId);
totalReplicaDataSizes.set(tabletOrderIdx,
-
totalReplicaDataSizes.get(tabletOrderIdx) + tablet.getDataSize(true));
+
totalReplicaDataSizes.get(tabletOrderIdx)
+ + tablet.getDataSize(true, false));
tabletOrderIdx++;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
index 8e5f46573ee..572db637f9e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/clone/DynamicPartitionScheduler.java
@@ -58,6 +58,7 @@ import org.apache.doris.persist.PartitionPersistInfo;
import org.apache.doris.rpc.RpcException;
import org.apache.doris.thrift.TStorageMedium;
+import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -159,7 +160,7 @@ public class DynamicPartitionScheduler extends MasterDaemon
{
}
// exponential moving average
- private static long ema(ArrayList<Long> history, int period) {
+ private static long ema(List<Long> history, int period) {
double alpha = 2.0 / (period + 1);
double ema = history.get(0);
for (int i = 1; i < history.size(); i++) {
@@ -168,7 +169,7 @@ public class DynamicPartitionScheduler extends MasterDaemon
{
return (long) ema;
}
- private static long getNextPartitionSize(ArrayList<Long>
historyPartitionsSize) {
+ private static long getNextPartitionSize(List<Long> historyPartitionsSize)
{
if (historyPartitionsSize.size() < 2) {
return historyPartitionsSize.get(0);
}
@@ -191,65 +192,98 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
}
}
- private static int getBucketsNum(DynamicPartitionProperty property,
OlapTable table,
+ private static Pair<Integer, Integer>
getBucketsNum(DynamicPartitionProperty property, OlapTable table,
String partitionName, String nowPartitionName, boolean
executeFirstTime) {
// if execute first time, all partitions no contain data
if (!table.isAutoBucket() || executeFirstTime) {
- return property.getBuckets();
+ return Pair.of(property.getBuckets(), 0);
}
- // auto bucket
- // get all history partitions
- RangePartitionInfo info = (RangePartitionInfo)
(table.getPartitionInfo());
- List<Map.Entry<Long, PartitionItem>> idToItems = new
ArrayList<>(info.getIdToItem(false).entrySet());
- idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem)
o.getValue()).getItems().upperEndpoint()));
- List<Partition> partitions = idToItems.stream()
- .map(entry -> table.getPartition(entry.getKey()))
- .filter(partition -> partition != null &&
!partition.getName().equals(nowPartitionName))
- .collect(Collectors.toList());
- List<Long> visibleVersions = null;
+ List<Partition> partitions = getHistoricalPartitions(table,
nowPartitionName);
+ List<Long> visibleVersions;
try {
visibleVersions = Partition.getVisibleVersions(partitions);
} catch (RpcException e) {
- LOG.warn("autobucket use property's buckets get visible version
fail, table: [{}-{}], "
+ LOG.warn("auto bucket use property's buckets get visible version
fail, table: [{}-{}], "
+ "partition: {}, buckets num: {}, exception: ",
table.getName(), table.getId(), partitionName,
property.getBuckets(), e);
- return property.getBuckets();
+ return Pair.of(property.getBuckets(), 0);
}
- List<Partition> hasDataPartitions = Lists.newArrayList();
+ List<Partition> hasDataPartitions = filterDataPartitions(partitions,
visibleVersions);
+ if (hasDataPartitions.isEmpty()) {
+ return handleNoDataPartitions(table, partitionName,
property.getBuckets());
+ }
+
+ return calculateBuckets(hasDataPartitions);
+ }
+
+ private static List<Partition> getHistoricalPartitions(OlapTable table,
String nowPartitionName) {
+ RangePartitionInfo info = (RangePartitionInfo)
(table.getPartitionInfo());
+ List<Map.Entry<Long, PartitionItem>> idToItems = new
ArrayList<>(info.getIdToItem(false).entrySet());
+ idToItems.sort(Comparator.comparing(o -> ((RangePartitionItem)
o.getValue()).getItems().upperEndpoint()));
+ return idToItems.stream()
+ .map(entry -> table.getPartition(entry.getKey()))
+ .filter(partition -> partition != null &&
!partition.getName().equals(nowPartitionName))
+ .collect(Collectors.toList());
+ }
+
+ private static List<Partition> filterDataPartitions(List<Partition>
partitions, List<Long> visibleVersions) {
+ Preconditions.checkState(partitions.size() == visibleVersions.size(),
+ String.format("partitions size %d not eq visibleVersions size
%d, impossible",
+ partitions.size(), visibleVersions.size()));
+ List<Partition> hasDataPartitions = new ArrayList<>();
for (int i = 0; i < partitions.size(); i++) {
if (visibleVersions.get(i) >= 2) {
hasDataPartitions.add(partitions.get(i));
}
}
+ return hasDataPartitions;
+ }
- // no exist history partition data
- if (hasDataPartitions.isEmpty()) {
- LOG.info("autobucket use property's buckets due to all partitions
no data, table: [{}-{}], "
- + "partition: {}, buckets num: {}",
- table.getName(), table.getId(), partitionName,
property.getBuckets());
- return property.getBuckets();
+ private static Pair<Integer, Integer> handleNoDataPartitions(OlapTable
table,
+ String
partitionName, int defaultBuckets) {
+ LOG.info("auto bucket use property's buckets due to all partitions no
data, table: [{}-{}], "
+ + "partition: {}, buckets num: {}", table.getName(),
table.getId(), partitionName, defaultBuckets);
+ return Pair.of(defaultBuckets, 0);
+ }
+
+ private static Pair<Integer, Integer> calculateBuckets(List<Partition>
hasDataPartitions) {
+ List<Long> partitionSizeArray = new ArrayList<>();
+ List<Long> sizeUnknownArray = new ArrayList<>();
+
+ for (Partition hasDataPartition : hasDataPartitions) {
+ long partitionSize =
hasDataPartition.getDataSizeExcludeEmptyReplica(true);
+ if (partitionSize <= 0) {
+ sizeUnknownArray.add(partitionSize);
+ } else {
+ partitionSizeArray.add(partitionSize);
+ }
+ }
+
+ int size = hasDataPartitions.size();
+ Preconditions.checkState(size > 0, "hasDataPartitions size must be
greater than 0");
+ int previousPartitionBucketsNum = hasDataPartitions.get(size -
1).getDistributionInfo().getBucketNum();
+
+ if (hasDataPartitions.size() == sizeUnknownArray.size()) {
+ LOG.info("TabletStatMgr not synchronized partitions size yet, so
use previous partition bucket num");
+ return Pair.of(previousPartitionBucketsNum,
previousPartitionBucketsNum);
}
- ArrayList<Long> partitionSizeArray = hasDataPartitions.stream()
- .map(partition -> partition.getAllDataSize(true))
- .collect(Collectors.toCollection(ArrayList::new));
long estimatePartitionSize = getNextPartitionSize(partitionSizeArray);
- // plus 5 for uncompressed data
long uncompressedPartitionSize = estimatePartitionSize * 5;
int bucketsNum =
AutoBucketUtils.getBucketsNum(uncompressedPartitionSize,
Config.autobucket_min_buckets);
- LOG.info("autobucket calc with {} history partitions, table: [{}-{}],
partition: {}, buckets num: {}, "
- + " estimate partition size: {}, last partitions(partition
name, local size, remote size): {}",
- hasDataPartitions.size(), table.getName(), table.getId(),
partitionName, bucketsNum,
- estimatePartitionSize,
+
+ LOG.info("auto bucket calc with {} history partitions, {} history
partitions size not sync size yet,"
+ + " buckets num: {}, estimate partition size: {}, last
partitions: {}",
+ hasDataPartitions.size(), sizeUnknownArray.size(), bucketsNum,
estimatePartitionSize,
hasDataPartitions.stream()
.skip(Math.max(0, hasDataPartitions.size() - 7))
.map(partition -> "(" + partition.getName() + ", " +
partition.getDataSize(true)
+ ", " + partition.getRemoteDataSize() + ")")
.collect(Collectors.toList()));
- return bucketsNum;
+ return Pair.of(bucketsNum, previousPartitionBucketsNum);
}
private ArrayList<AddPartitionClause> getAddPartitionClause(Database db,
OlapTable olapTable,
@@ -356,8 +390,17 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
DistributionDesc distributionDesc = null;
DistributionInfo distributionInfo =
olapTable.getDefaultDistributionInfo();
- int bucketsNum = getBucketsNum(dynamicPartitionProperty,
olapTable, partitionName,
+ Pair<Integer, Integer> ret =
getBucketsNum(dynamicPartitionProperty, olapTable, partitionName,
nowPartitionName, executeFirstTime);
+ int bucketsNum = ret.first;
+ int previousPartitionBucketsNum = ret.second;
+ if (olapTable.isAutoBucket()) {
+ int afterCheckAndFixBucketNum =
checkAndFixAutoBucketCalcNumIsValid(bucketsNum,
+ previousPartitionBucketsNum);
+ if (afterCheckAndFixBucketNum > 0) {
+ bucketsNum = afterCheckAndFixBucketNum;
+ }
+ }
if (distributionInfo.getType() ==
DistributionInfo.DistributionInfoType.HASH) {
HashDistributionInfo hashDistributionInfo =
(HashDistributionInfo) distributionInfo;
List<String> distColumnNames = new ArrayList<>();
@@ -374,6 +417,33 @@ public class DynamicPartitionScheduler extends
MasterDaemon {
return addPartitionClauses;
}
+ private int checkAndFixAutoBucketCalcNumIsValid(int
currentPartitionNumBuckets, int previousPartitionNumBuckets) {
+ // previousPartitionBucketsNum == 0, some abnormal case, ignore it
+ if (currentPartitionNumBuckets != 0) {
+ // currentPartitionNumBuckets can be too big
+ if (currentPartitionNumBuckets
+ > previousPartitionNumBuckets * (1 +
Config.autobucket_out_of_bounds_percent_threshold)) {
+ LOG.warn("auto bucket calc num may be err, bigger than
previous too much, plz check. "
+ + "calc bucket num {}, previous partition bucket num
{}, percent {}",
+ currentPartitionNumBuckets,
previousPartitionNumBuckets,
+ Config.autobucket_out_of_bounds_percent_threshold);
+ return currentPartitionNumBuckets;
+ }
+ // currentPartitionNumBuckets not too small.
+ // If it is too small, the program will intervene. use
previousPartitionNumBuckets
+ if (currentPartitionNumBuckets
+ < previousPartitionNumBuckets * (1 -
Config.autobucket_out_of_bounds_percent_threshold)) {
+ LOG.warn("auto bucket calc num may be err, smaller than
previous too much, plz check. "
+ + "calc bucket num {}, previous partition bucket num
{}, percent {}",
+ currentPartitionNumBuckets,
previousPartitionNumBuckets,
+ Config.autobucket_out_of_bounds_percent_threshold);
+ return previousPartitionNumBuckets;
+ }
+ }
+ LOG.info("previousPartitionBucketsNum eq 0, check before log");
+ return -1;
+ }
+
/**
* If dynamic_partition.storage_medium is set to SSD,
* ignore hot_partition_num property and set to (SSD, 9999-12-31 23:59:59)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java
index 6b7b0e5282a..6797ce45acb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/CacheHotspotManager.java
@@ -501,13 +501,13 @@ public class CacheHotspotManager extends MasterDaemon {
List<Long> batch = new ArrayList<>();
long curBatchSize = 0L;
for (Tablet tablet : entry.getValue()) {
- if (curBatchSize + tablet.getDataSize(true) > maxSizePerBatch)
{
+ if (curBatchSize + tablet.getDataSize(true, false) >
maxSizePerBatch) {
batches.add(batch);
batch = new ArrayList<>();
curBatchSize = 0L;
}
batch.add(tablet.getId());
- curBatchSize += tablet.getDataSize(true);
+ curBatchSize += tablet.getDataSize(true, false);
}
if (!batch.isEmpty()) {
batches.add(batch);
@@ -545,7 +545,7 @@ public class CacheHotspotManager extends MasterDaemon {
continue;
}
for (Tablet tablet : index.getTablets()) {
- warmUpTabletsSize += tablet.getDataSize(true);
+ warmUpTabletsSize += tablet.getDataSize(true, false);
tablets.add(tablet);
if (warmUpTabletsSize >= dstTotalFileCache) {
break;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
index 6c36a926d08..b1be9c79ab6 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletHealthProcDir.java
@@ -262,7 +262,7 @@ public class TabletHealthProcDir implements
ProcDirInterface {
inconsistentNum++;
inconsistentTabletIds.add(tablet.getId());
}
- if (tablet.getDataSize(true) >
Config.min_bytes_indicate_replica_too_large) {
+ if (tablet.getDataSize(true, false) >
Config.min_bytes_indicate_replica_too_large) {
oversizeNum++;
oversizeTabletIds.add(tablet.getId());
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
index b16d3d15cf7..8a19912d321 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CatalogTestUtil.java
@@ -385,7 +385,7 @@ public class CatalogTestUtil {
if (tablet == null) {
return -1L;
}
- return tablet.getDataSize(true);
+ return tablet.getDataSize(true, false);
} finally {
olapTable.readUnlock();
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
index 9495c048f4b..ad5b1ce0555 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DynamicPartitionTableTest.java
@@ -1779,6 +1779,7 @@ public class DynamicPartitionTableTest {
}
RebalancerTestUtil.updateReplicaDataSize(1, 1, 1);
+ Config.autobucket_out_of_bounds_percent_threshold = 0.99;
String alterStmt1 =
"alter table test.test_autobucket_dynamic_partition set
('dynamic_partition.end' = '2')";
ExceptionChecker.expectThrowsNoException(() -> alterTable(alterStmt1));
@@ -1789,7 +1790,7 @@ public class DynamicPartitionTableTest {
partitions.sort(Comparator.comparing(Partition::getId));
Assert.assertEquals(53, partitions.size());
Assert.assertEquals(1, partitions.get(partitions.size() -
1).getDistributionInfo().getBucketNum());
-
+ Config.autobucket_out_of_bounds_percent_threshold = 0.5;
table.readLock();
try {
// first 40 partitions with size 0, then 13 partitions with size
100GB(10GB * 10 buckets)
@@ -1826,5 +1827,48 @@ public class DynamicPartitionTableTest {
Assert.assertEquals(54, partitions.size());
// 100GB total, 5GB per bucket, should 20 buckets.
Assert.assertEquals(20, partitions.get(partitions.size() -
1).getDistributionInfo().getBucketNum());
+
+ // mock partition size eq 0, use back-to-back logic
+ table.readLock();
+ try {
+ // when fe restart, when stat thread not get replica size from
be/ms, replica size eq 0
+ for (int i = 0; i < 54; i++) {
+ Partition partition = partitions.get(i);
+ partition.updateVisibleVersion(2L);
+ for (MaterializedIndex idx : partition.getMaterializedIndices(
+ MaterializedIndex.IndexExtState.VISIBLE)) {
+ if (i < 52) {
+ Assert.assertEquals(10, idx.getTablets().size());
+ } else if (i == 52) {
+ Assert.assertEquals(1, idx.getTablets().size());
+ } else if (i == 53) {
+ Assert.assertEquals(20, idx.getTablets().size());
+ }
+ for (Tablet tablet : idx.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ replica.updateVersion(3L);
+ // mock replica size eq 0
+ replica.setDataSize(0L);
+ replica.setRowCount(0L);
+ }
+ }
+ }
+ Assert.assertEquals(0, partition.getAllDataSize(true));
+ }
+ } finally {
+ table.readUnlock();
+ }
+
+ String alterStmt3 = "alter table
test.test_autobucket_dynamic_partition set ('dynamic_partition.end' = '4')";
+ ExceptionChecker.expectThrowsNoException(() -> alterTable(alterStmt3));
+ // 54th previous partition size set 53, check back to back logic work
+ partitions.get(53).getDistributionInfo().setBucketNum(53);
+
Env.getCurrentEnv().getDynamicPartitionScheduler().executeDynamicPartition(tempDynamicPartitionTableInfo,
false);
+
+ partitions = Lists.newArrayList(table.getAllPartitions());
+ partitions.sort(Comparator.comparing(Partition::getId));
+ Assert.assertEquals(55, partitions.size());
+ // due to partition size eq 0, use previous partition's(54th) bucket
num
+ Assert.assertEquals(53, partitions.get(partitions.size() -
1).getDistributionInfo().getBucketNum());
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
index bd3e7f9e2a5..f4f364b91ce 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/AutoBucketUtilsTest.java
@@ -138,6 +138,7 @@ public class AutoBucketUtilsTest {
Config.tablet_checker_interval_ms = 1000;
Config.tablet_repair_delay_factor_second = 1;
connectContext = UtFrameUtils.createDefaultCtx();
+ Config.autobucket_partition_size_per_bucket_gb = 1;
}
@After
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
