This is an automated email from the ASF dual-hosted git repository.
hello-stephen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 17617be150e [fix] (cloud) Fix local/remote tablet size semantics in
schema views (#60887)
17617be150e is described below
commit 17617be150e43fb749cf77cd55adc0b08a2f3afb
Author: deardeng <[email protected]>
AuthorDate: Mon Jun 1 10:38:23 2026 +0800
[fix] (cloud) Fix local/remote tablet size semantics in schema views
(#60887)
In storage-compute separation, data size should be represented
consistently as remote size.
Previously, show tablets and information_schema.partitions could diverge
from information_schema.backend_tablets, which made local/remote
semantics confusing for users and operators.
This change aligns cloud-mode output mapping for local/remote size
columns and adds a regression test to guard the behavior.
---
.../apache/doris/catalog/DataSizeDisplayUtil.java | 78 ++++++++++++
.../apache/doris/common/proc/TabletsProcDir.java | 9 +-
.../doris/tablefunction/MetadataGenerator.java | 18 ++-
.../doris/catalog/DataSizeDisplayUtilTest.java | 124 +++++++++++++++++++
.../tablets/test_tablet_size_semantics.groovy | 131 +++++++++++++++++++++
5 files changed, 354 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java
new file mode 100644
index 00000000000..03532203409
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
+
+public class DataSizeDisplayUtil {
+ private DataSizeDisplayUtil() {
+ }
+
+ public static Pair<Long, Long> getDisplayDataSize(Replica replica) {
+ return getDisplayDataSize(replica.getDataSize(),
replica.getRemoteDataSize(),
+ getLocalIndexAndSegmentSize(replica));
+ }
+
+ public static Pair<Long, Long> getDisplayDataSize(Partition partition) {
+ long localDataSize = partition.getDataSize(false);
+ long remoteDataSize = partition.getRemoteDataSize();
+ if (!needCloudSizeMapping(remoteDataSize)) {
+ return Pair.of(localDataSize, remoteDataSize);
+ }
+ return getPartitionDisplayDataSize(partition);
+ }
+
+ private static Pair<Long, Long> getDisplayDataSize(long localDataSize,
long remoteDataSize,
+ long localIndexAndSegmentSize) {
+ if (!needCloudSizeMapping(remoteDataSize)) {
+ return Pair.of(localDataSize, remoteDataSize);
+ }
+ if (localDataSize > 0) {
+ remoteDataSize = localDataSize;
+ localDataSize = 0;
+ } else if (localIndexAndSegmentSize > 0) {
+ remoteDataSize = localIndexAndSegmentSize;
+ }
+ return Pair.of(localDataSize, remoteDataSize);
+ }
+
+ private static boolean needCloudSizeMapping(long remoteDataSize) {
+ return Config.isCloudMode() && remoteDataSize == 0;
+ }
+
+ private static Pair<Long, Long> getPartitionDisplayDataSize(Partition
partition) {
+ long localDataSize = 0L;
+ long remoteDataSize = 0L;
+ for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+ for (Tablet tablet : index.getTablets()) {
+ for (Replica replica : tablet.getReplicas()) {
+ Pair<Long, Long> displayDataSize =
getDisplayDataSize(replica);
+ localDataSize += displayDataSize.first;
+ remoteDataSize += displayDataSize.second;
+ }
+ }
+ }
+ return Pair.of(localDataSize, remoteDataSize);
+ }
+
+ private static long getLocalIndexAndSegmentSize(Replica replica) {
+ return replica.getLocalInvertedIndexSize() +
replica.getLocalSegmentSize();
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
index ce3ab7a926b..8a8408a91c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java
@@ -18,6 +18,7 @@
package org.apache.doris.common.proc;
import org.apache.doris.catalog.CloudTabletStatMgr;
+import org.apache.doris.catalog.DataSizeDisplayUtil;
import org.apache.doris.catalog.DiskInfo;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
@@ -29,6 +30,7 @@ import org.apache.doris.cloud.catalog.CloudReplica;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.util.ListComparator;
import org.apache.doris.common.util.NetUtils;
import org.apache.doris.common.util.TimeUtils;
@@ -193,8 +195,11 @@ public class TabletsProcDir implements ProcDirInterface {
tabletInfo.add(displayLastSuccessVersion);
tabletInfo.add(replica.getLastFailedVersion());
tabletInfo.add(TimeUtils.longToTimeString(replica.getLastFailedTimestamp()));
- tabletInfo.add(replica.getDataSize());
- tabletInfo.add(replica.getRemoteDataSize());
+ Pair<Long, Long> displayDataSize =
DataSizeDisplayUtil.getDisplayDataSize(replica);
+ long localDataSize = displayDataSize.first;
+ long remoteDataSize = displayDataSize.second;
+ tabletInfo.add(localDataSize);
+ tabletInfo.add(remoteDataSize);
tabletInfo.add(replica.getRowCount());
tabletInfo.add(replica.getState());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
index fa82795e065..85911b3f3ae 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java
@@ -23,6 +23,7 @@ import org.apache.doris.authentication.RoleMappingMeta;
import org.apache.doris.blockrule.SqlBlockRule;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
+import org.apache.doris.catalog.DataSizeDisplayUtil;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DistributionInfo;
@@ -46,6 +47,7 @@ import org.apache.doris.catalog.View;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.Pair;
import org.apache.doris.common.proc.FrontendsProcNode;
import org.apache.doris.common.proc.PartitionsProcDir;
@@ -1913,17 +1915,21 @@ public class MetadataGenerator {
trow.addToColumnValue(new TCell().setStringVal("")); //
NODEGROUP (not available)
trow.addToColumnValue(new TCell().setStringVal("")); //
TABLESPACE_NAME (not available)
- Pair<Double, String> sizePair =
DebugUtil.getByteUint(partition.getDataSize(false));
+ Pair<Long, Long> displayDataSize =
DataSizeDisplayUtil.getDisplayDataSize(partition);
+ long localDataSize = displayDataSize.first;
+ long remoteDataSize = displayDataSize.second;
+ Pair<Double, String> sizePair =
DebugUtil.getByteUint(localDataSize);
String readableDateSize =
DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
+ sizePair.second;
trow.addToColumnValue(new
TCell().setStringVal(readableDateSize)); // LOCAL_DATA_SIZE
- sizePair =
DebugUtil.getByteUint(partition.getRemoteDataSize());
+ sizePair = DebugUtil.getByteUint(remoteDataSize);
readableDateSize =
DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
+ sizePair.second;
trow.addToColumnValue(new
TCell().setStringVal(readableDateSize)); // REMOTE_DATA_SIZE
trow.addToColumnValue(new
TCell().setStringVal(partition.getState().toString())); // STATE
- String replicaAllocation =
PartitionsProcDir.getReplicaAllocationDisplay(
-
partitionInfo.getReplicaAllocation(partitionId).toCreateStmt());
+ String replicaAllocation =
getPartitionsReplicaAllocationDisplay(
+
PartitionsProcDir.getReplicaAllocationDisplay(partitionInfo.getReplicaAllocation(
+ partitionId).toCreateStmt()));
trow.addToColumnValue(new
TCell().setStringVal(replicaAllocation)); // REPLICA_ALLOCATION
trow.addToColumnValue(new
TCell().setIntVal(partitionInfo.getReplicaAllocation(partitionId)
.getTotalReplicaNum())); // REPLICA_NUM
@@ -1988,6 +1994,10 @@ public class MetadataGenerator {
} // for table
}
+ private static String getPartitionsReplicaAllocationDisplay(String
replicaAllocation) {
+ return FeConstants.null_string.equals(replicaAllocation) ? "NULL" :
replicaAllocation;
+ }
+
private static void partitionsForExternalCatalog(UserIdentity
currentUserIdentity,
CatalogIf catalog, DatabaseIf database, List<TableIf> tables,
List<TRow> dataBatch, String timeZone) {
for (TableIf table : tables) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java
new file mode 100644
index 00000000000..d88397e977a
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java
@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.catalog.MaterializedIndex.IndexState;
+import org.apache.doris.cloud.catalog.CloudReplica;
+import org.apache.doris.cloud.catalog.CloudTablet;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DataSizeDisplayUtilTest {
+ private String originDeployMode;
+ private String originCloudUniqueId;
+
+ @Before
+ public void setUp() {
+ originDeployMode = Config.deploy_mode;
+ originCloudUniqueId = Config.cloud_unique_id;
+ Config.deploy_mode = "cloud";
+ Config.cloud_unique_id = "";
+ }
+
+ @After
+ public void tearDown() {
+ Config.deploy_mode = originDeployMode;
+ Config.cloud_unique_id = originCloudUniqueId;
+ }
+
+ @Test
+ public void testPartitionDisplaySizeFallbackToReplicaIndexAndSegmentSize()
{
+ MaterializedIndex baseIndex = new MaterializedIndex(10L,
IndexState.NORMAL);
+ CloudTablet tablet = new CloudTablet(20L);
+ CloudReplica replica = new CloudReplica(30L, 1L,
Replica.ReplicaState.NORMAL, 2L, 0,
+ 100L, 200L, 300L, 10L, 0L);
+ replica.setDataSize(0L);
+ replica.setLocalInvertedIndexSize(111L);
+ replica.setLocalSegmentSize(222L);
+ tablet.addReplica(replica, true);
+ baseIndex.addTablet(tablet, null, true);
+
+ Partition partition = new Partition(300L, "p1", baseIndex, new
RandomDistributionInfo(1));
+
+ Pair<Long, Long> displayDataSize =
DataSizeDisplayUtil.getDisplayDataSize(partition);
+ Assert.assertEquals(0L, displayDataSize.first.longValue());
+ Assert.assertEquals(333L, displayDataSize.second.longValue());
+ }
+
+ @Test
+ public void testPartitionDisplaySizeMapsCloudDataSizeToRemoteSize() {
+ MaterializedIndex baseIndex = new MaterializedIndex(10L,
IndexState.NORMAL);
+ CloudTablet tablet = new CloudTablet(20L);
+ CloudReplica replica = new CloudReplica(30L, 1L,
Replica.ReplicaState.NORMAL, 2L, 0,
+ 100L, 200L, 300L, 10L, 0L);
+ replica.setDataSize(123L);
+ tablet.addReplica(replica, true);
+ baseIndex.addTablet(tablet, null, true);
+
+ Partition partition = new Partition(300L, "p1", baseIndex, new
RandomDistributionInfo(1));
+
+ Pair<Long, Long> displayDataSize =
DataSizeDisplayUtil.getDisplayDataSize(partition);
+ Assert.assertEquals(0L, displayDataSize.first.longValue());
+ Assert.assertEquals(123L, displayDataSize.second.longValue());
+ }
+
+ @Test
+ public void testPartitionDisplaySizeAggregatesMixedReplicaDisplaySize() {
+ MaterializedIndex baseIndex = new MaterializedIndex(10L,
IndexState.NORMAL);
+
+ CloudTablet tabletWithDataSize = new CloudTablet(20L);
+ CloudReplica replicaWithDataSize = new CloudReplica(30L, 1L,
Replica.ReplicaState.NORMAL, 2L, 0,
+ 100L, 200L, 300L, 10L, 0L);
+ replicaWithDataSize.setDataSize(123L);
+ tabletWithDataSize.addReplica(replicaWithDataSize, true);
+ baseIndex.addTablet(tabletWithDataSize, null, true);
+
+ CloudTablet tabletWithFallbackSize = new CloudTablet(21L);
+ CloudReplica replicaWithFallbackSize = new CloudReplica(31L, 1L,
Replica.ReplicaState.NORMAL, 2L, 0,
+ 100L, 200L, 300L, 10L, 1L);
+ replicaWithFallbackSize.setDataSize(0L);
+ replicaWithFallbackSize.setLocalInvertedIndexSize(111L);
+ replicaWithFallbackSize.setLocalSegmentSize(222L);
+ tabletWithFallbackSize.addReplica(replicaWithFallbackSize, true);
+ baseIndex.addTablet(tabletWithFallbackSize, null, true);
+
+ Partition partition = new Partition(300L, "p1", baseIndex, new
RandomDistributionInfo(2));
+
+ Pair<Long, Long> displayDataSize =
DataSizeDisplayUtil.getDisplayDataSize(partition);
+ Assert.assertEquals(0L, displayDataSize.first.longValue());
+ Assert.assertEquals(456L, displayDataSize.second.longValue());
+ }
+
+ @Test
+ public void testReplicaDisplaySizeFallbackToReplicaIndexAndSegmentSize() {
+ CloudReplica replica = new CloudReplica(30L, 1L,
Replica.ReplicaState.NORMAL, 2L, 0,
+ 100L, 200L, 300L, 10L, 0L);
+ replica.setDataSize(0L);
+ replica.setLocalInvertedIndexSize(111L);
+ replica.setLocalSegmentSize(222L);
+
+ Pair<Long, Long> displayDataSize =
DataSizeDisplayUtil.getDisplayDataSize(replica);
+ Assert.assertEquals(0L, displayDataSize.first.longValue());
+ Assert.assertEquals(333L, displayDataSize.second.longValue());
+ }
+}
diff --git
a/regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy
b/regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy
new file mode 100644
index 00000000000..673800f97f6
--- /dev/null
+++ b/regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_tablet_size_semantics") {
+ if (!isCloudMode()) {
+ return
+ }
+
+ def dbName = "test_tablet_size_semantics_db"
+ def tableName = "test_tablet_size_semantics_tbl"
+ def parseDataSizeToBytes = { Object dataSize ->
+ String normalized = dataSize == null ? "" :
dataSize.toString().replaceAll("\\s+", "")
+ if (normalized.isEmpty()) {
+ return 0L
+ }
+ if (!(normalized ==~ /.*[A-Za-z]$/)) {
+ normalized = normalized + "B"
+ }
+ def parsedRows = sql_return_maparray("SELECT
parse_data_size('${normalized}') AS bytes")
+ assertTrue(parsedRows.size() == 1)
+ return parsedRows[0]["bytes"].toString().toLong()
+ }
+
+ sql "drop database if exists ${dbName}"
+ sql "create database ${dbName}"
+ sql "use ${dbName}"
+
+ sql """
+ CREATE TABLE IF NOT EXISTS `${tableName}` (
+ `k1` INT NOT NULL,
+ `v1` STRING NULL
+ )
+ DUPLICATE KEY(`k1`)
+ DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+ PROPERTIES (
+ "replication_num" = "1"
+ );
+ """
+
+ sql """
+ INSERT INTO `${tableName}` VALUES
+ (1, 'a'),
+ (2, 'b'),
+ (3, 'c'),
+ (4, 'd'),
+ (5, 'e');
+ """
+ sql "sync"
+
+ def showTabletRows = sql_return_maparray("show tablets from ${tableName}")
+ assertTrue(showTabletRows.size() == 1)
+ def showTabletRow = showTabletRows[0]
+ long tabletId = showTabletRow["TabletId"].toString().toLong()
+ long showLocalSize = -1L
+ long showRemoteSize = -1L
+ long backendLocalSize = -1L
+ long backendRemoteSize = -1L
+ boolean sizeAligned = false
+
+ // SHOW TABLETS size stats may lag behind backend_tablets for tens of
seconds.
+ for (int i = 0; i < 120; i++) {
+ def latestShowTabletRows = sql_return_maparray("show tablets from
${tableName}")
+ assertTrue(latestShowTabletRows.size() == 1)
+ def latestShowTabletRow = latestShowTabletRows[0]
+ showLocalSize =
latestShowTabletRow["LocalDataSize"].toString().toLong()
+ showRemoteSize =
latestShowTabletRow["RemoteDataSize"].toString().toLong()
+ long backendIdForCheck =
latestShowTabletRow["BackendId"].toString().toLong()
+
+ def backendTabletRows = sql_return_maparray("""
+ SELECT
+ tablet_local_size AS be_local_size,
+ tablet_remote_size AS be_remote_size
+ FROM information_schema.backend_tablets
+ WHERE tablet_id = ${tabletId}
+ AND be_id = ${backendIdForCheck}
+ ORDER BY update_time DESC
+ LIMIT 1
+ """)
+ if (backendTabletRows.size() == 1) {
+ def backendTabletRow = backendTabletRows[0]
+ backendLocalSize =
backendTabletRow["be_local_size"].toString().toLong()
+ backendRemoteSize =
backendTabletRow["be_remote_size"].toString().toLong()
+ if (showLocalSize == backendLocalSize
+ && showRemoteSize == backendRemoteSize
+ && backendRemoteSize > 0) {
+ sizeAligned = true
+ break
+ }
+ }
+
+ logger.info("wait tablet size aligned, show local/remote: {}/{},
backend local/remote: {}/{}",
+ showLocalSize, showRemoteSize, backendLocalSize,
backendRemoteSize)
+ sleep(1000)
+ }
+ assertTrue(sizeAligned,
+ "tablet size not aligned within 120s, show
local/remote=${showLocalSize}/${showRemoteSize}, " +
+ "backend
local/remote=${backendLocalSize}/${backendRemoteSize}")
+ logger.info("final tablet size aligned, show local/remote: {}/{}, backend
local/remote: {}/{}",
+ showLocalSize, showRemoteSize, backendLocalSize,
backendRemoteSize)
+ def partitionRows = sql_return_maparray("""
+ SELECT
+ local_data_size AS part_local_size,
+ remote_data_size AS part_remote_size
+ FROM information_schema.partitions
+ WHERE table_schema = '${dbName}'
+ AND table_name = '${tableName}'
+ ORDER BY partition_name
+ LIMIT 1
+ """)
+ assertTrue(partitionRows.size() == 1)
+ def partitionRow = partitionRows[0]
+ long partitionLocalSize =
parseDataSizeToBytes(partitionRow["part_local_size"])
+ long partitionRemoteSize =
parseDataSizeToBytes(partitionRow["part_remote_size"])
+
+ assertEquals(0L, partitionLocalSize)
+ assertTrue(partitionRemoteSize > 0)
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
