This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 191910ee0bc9497a73d54b5748526ef3898cb207
Author: Jibing-Li <64681310+jibing...@users.noreply.github.com>
AuthorDate: Fri Jul 19 10:38:04 2024 +0800
[improvement](statistics) Update table/index row count in one operation.
(#38059)
Atomic update table/index row count. Avoid the row count in partial
updated status.
---
.../main/java/org/apache/doris/analysis/ShowDataStmt.java | 10 ++++++----
.../java/org/apache/doris/catalog/CloudTabletStatMgr.java | 7 ++++---
.../src/main/java/org/apache/doris/catalog/OlapTable.java | 15 +++++++++------
.../main/java/org/apache/doris/catalog/TabletStatMgr.java | 5 ++++-
.../java/org/apache/doris/analysis/ShowDataStmtTest.java | 2 +-
regression-test/suites/statistics/test_analyze_mv.groovy | 9 ++++++++-
6 files changed, 32 insertions(+), 16 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
index 51690f9ce87..b87c0f0788f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
@@ -90,6 +90,7 @@ public class ShowDataStmt extends ShowStmt {
.addColumn(new Column("ReplicaCount",
ScalarType.createVarchar(20)))
.addColumn(new Column("RowCount",
ScalarType.createVarchar(20)))
.addColumn(new Column("RemoteSize",
ScalarType.createVarchar(30)))
+ .addColumn(new Column("StatsRowCount",
ScalarType.createVarchar(20)))
.build();
public static final ImmutableList<String> SHOW_TABLE_DATA_META_DATA_ORIGIN
=
@@ -288,6 +289,7 @@ public class ShowDataStmt extends ShowStmt {
long indexReplicaCount = 0;
long indexRowCount = 0;
long indexRemoteSize = 0;
+ long indexStatsRowCount =
olapTable.getRowCountForIndex(indexId);
for (Partition partition : olapTable.getAllPartitions()) {
MaterializedIndex mIndex = partition.getIndex(indexId);
indexSize += mIndex.getDataSize(false);
@@ -300,7 +302,7 @@ public class ShowDataStmt extends ShowStmt {
//
.add("TableName").add("IndexName").add("Size").add("ReplicaCount").add("RowCount")
// .add("RemoteSize")
List<Object> row = Arrays.asList(tableName, indexName,
indexSize, indexReplicaCount,
- indexRowCount, indexRemoteSize);
+ indexRowCount, indexRemoteSize,
indexStatsRowCount);
totalRowsObject.add(row);
totalSize += indexSize;
@@ -333,11 +335,11 @@ public class ShowDataStmt extends ShowStmt {
if (index == 0) {
result = Arrays.asList(tableName.getTbl(),
String.valueOf(row.get(1)),
readableSize, String.valueOf(row.get(3)),
- String.valueOf(row.get(4)),
remoteReadableSize);
+ String.valueOf(row.get(4)),
remoteReadableSize, String.valueOf(row.get(6)));
} else {
result = Arrays.asList("", String.valueOf(row.get(1)),
readableSize, String.valueOf(row.get(3)),
- String.valueOf(row.get(4)),
remoteReadableSize);
+ String.valueOf(row.get(4)),
remoteReadableSize, String.valueOf(row.get(6)));
}
totalRows.add(result);
}
@@ -349,7 +351,7 @@ public class ShowDataStmt extends ShowStmt {
String remoteReadableSize =
DebugUtil.DECIMAL_FORMAT_SCALE_3.format(totalRemoteSizePair.first) + " "
+ totalRemoteSizePair.second;
List<String> row = Arrays.asList("", "Total", readableSize,
String.valueOf(totalReplicaCount), "",
- remoteReadableSize);
+ remoteReadableSize, "");
totalRows.add(row);
} finally {
olapTable.readUnlock();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
index 3a75862f8d3..e762f678b79 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CloudTabletStatMgr.java
@@ -159,7 +159,7 @@ public class CloudTabletStatMgr extends MasterDaemon {
if (!table.writeLockIfExist()) {
continue;
}
-
+ Map<Long, Long> indexesRowCount = new HashMap<>();
try {
for (Partition partition : olapTable.getAllPartitions()) {
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
@@ -201,12 +201,13 @@ public class CloudTabletStatMgr extends MasterDaemon {
tableSegmentCount += tabletSegmentCount;
} // end for tablets
index.setRowCount(indexRowCount);
+ indexesRowCount.put(index.getId(), indexRowCount);
} // end for indices
} // end for partitions
olapTable.setStatistics(new
OlapTable.Statistics(db.getName(),
table.getName(), tableDataSize,
tableTotalReplicaDataSize, 0L,
- tableReplicaCount, tableRowCount,
tableRowsetCount, tableSegmentCount));
+ tableReplicaCount, tableRowCount,
tableRowsetCount, tableSegmentCount, indexesRowCount));
LOG.debug("finished to set row num for table: {} in
database: {}",
table.getName(), db.getFullName());
} finally {
@@ -215,7 +216,7 @@ public class CloudTabletStatMgr extends MasterDaemon {
newCloudTableStatsMap.put(Pair.of(dbId, table.getId()), new
OlapTable.Statistics(db.getName(),
table.getName(), tableDataSize,
tableTotalReplicaDataSize, 0L,
- tableReplicaCount, tableRowCount, tableRowsetCount,
tableSegmentCount));
+ tableReplicaCount, tableRowCount, tableRowsetCount,
tableSegmentCount, indexesRowCount));
}
}
this.cloudTableStatsMap = newCloudTableStatsMap;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 62b93ca8adc..92b6aa236ad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1436,12 +1436,11 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
}
public long getRowCountForIndex(long indexId) {
- long rowCount = 0;
- for (Map.Entry<Long, Partition> entry : idToPartition.entrySet()) {
- MaterializedIndex index = entry.getValue().getIndex(indexId);
- rowCount += index == null ? 0 : index.getRowCount();
+ Map<Long, Long> indexesRowCount = statistics.getIndexesRowCount();
+ if (indexesRowCount == null || !indexesRowCount.containsKey(indexId)) {
+ return -1;
}
- return rowCount;
+ return indexesRowCount.get(indexId);
}
@Override
@@ -2996,6 +2995,9 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
@Getter
private Long segmentCount;
+ @Getter
+ private Map<Long, Long> indexesRowCount;
+
public Statistics() {
this.dbName = null;
this.tableName = null;
@@ -3016,7 +3018,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
public Statistics(String dbName, String tableName,
Long dataSize, Long totalReplicaDataSize,
Long remoteDataSize, Long replicaCount, Long rowCount,
- Long rowsetCount, Long segmentCount) {
+ Long rowsetCount, Long segmentCount, Map<Long, Long>
indexesRowCount) {
this.dbName = dbName;
this.tableName = tableName;
@@ -3031,6 +3033,7 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf, GsonPostProc
this.rowCount = rowCount;
this.rowsetCount = rowsetCount;
this.segmentCount = segmentCount;
+ this.indexesRowCount = indexesRowCount;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index bf24d2bb390..ea0081e680a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -31,6 +31,7 @@ import com.google.common.collect.ImmutableMap;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ForkJoinPool;
@@ -123,6 +124,7 @@ public class TabletStatMgr extends MasterDaemon {
if (!table.tryWriteLockIfExist(3000, TimeUnit.MILLISECONDS)) {
continue;
}
+ Map<Long, Long> indexesRowCount = new HashMap<>();
try {
for (Partition partition : olapTable.getAllPartitions()) {
long version = partition.getVisibleVersion();
@@ -159,12 +161,13 @@ public class TabletStatMgr extends MasterDaemon {
indexRowCount += tabletRowCount;
} // end for tablets
index.setRowCount(indexRowCount);
+ indexesRowCount.put(index.getId(), indexRowCount);
} // end for indices
} // end for partitions
olapTable.setStatistics(new
OlapTable.Statistics(db.getName(), table.getName(),
tableDataSize, tableTotalReplicaDataSize,
- tableRemoteDataSize, tableReplicaCount,
tableRowCount, 0L, 0L));
+ tableRemoteDataSize, tableReplicaCount,
tableRowCount, 0L, 0L, indexesRowCount));
if (LOG.isDebugEnabled()) {
LOG.debug("finished to set row num for table: {} in
database: {}",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java
b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java
index 9b421de0c39..c52bb643b4c 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowDataStmtTest.java
@@ -156,7 +156,7 @@ public class ShowDataStmtTest {
Assert.assertEquals(
"SHOW DATA FROM `testDb`.`test_tbl` ORDER BY `ReplicaCount`
DESC, `Size` DESC",
stmt.toString());
- Assert.assertEquals(6, stmt.getMetaData().getColumnCount());
+ Assert.assertEquals(7, stmt.getMetaData().getColumnCount());
Assert.assertEquals(true, stmt.hasTable());
stmt = new ShowDataStmt(null, Arrays.asList(orderByElementOne,
orderByElementTwo), null);
diff --git a/regression-test/suites/statistics/test_analyze_mv.groovy
b/regression-test/suites/statistics/test_analyze_mv.groovy
index a5d5e17a55a..7e456d01f51 100644
--- a/regression-test/suites/statistics/test_analyze_mv.groovy
+++ b/regression-test/suites/statistics/test_analyze_mv.groovy
@@ -127,6 +127,10 @@ suite("test_analyze_mv") {
"replication_num" = "1"
)
"""
+ def rowCounts = sql """show data from mvTestDup"""
+ logger.info("row count: " + rowCounts)
+ assertEquals("0", rowCounts[0][4])
+ assertEquals("-1", rowCounts[0][6])
createMV("create materialized view mv1 as select key1 from mvTestDup;")
createMV("create materialized view mv2 as select key2 from mvTestDup;")
createMV("create materialized view mv3 as select key1, key2, sum(value1),
max(value2), min(value3) from mvTestDup group by key1, key2;")
@@ -233,7 +237,6 @@ suite("test_analyze_mv") {
"replication_num" = "1"
);
"""
-
createMV("create materialized view mv1 as select key2 from mvTestAgg;")
createMV("create materialized view mv3 as select key1, key2, sum(value1),
max(value2), min(value3) from mvTestAgg group by key1, key2;")
createMV("create materialized view mv6 as select key1, sum(value1) from
mvTestAgg group by key1;")
@@ -425,6 +428,10 @@ suite("test_analyze_mv") {
logger.info(e.getMessage());
return;
}
+ wait_row_count_reported("test_analyze_mv", "mvTestDup", 0, 6, "6")
+ wait_row_count_reported("test_analyze_mv", "mvTestDup", 1, 6, "6")
+ wait_row_count_reported("test_analyze_mv", "mvTestDup", 2, 6, "4")
+ wait_row_count_reported("test_analyze_mv", "mvTestDup", 3, 6, "6")
sql """analyze table mvTestDup with sample rows 4000000"""
wait_analyze_finish("mvTestDup")
result_sample = sql """SHOW ANALYZE mvTestDup;"""
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
