This is an automated email from the ASF dual-hosted git repository.
lide pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new 367d2642dd2 [optimize](table stat) Reduce lock when get table
statistics (#39807)
367d2642dd2 is described below
commit 367d2642dd2821e1285d9d55f132d0757fab56c9
Author: xy720 <22125576+xy...@users.noreply.github.com>
AuthorDate: Wed Aug 28 16:42:37 2024 +0800
[optimize](table stat) Reduce lock when get table statistics (#39807)
## Proposed changes
Issue Number: this pr will pick part of #35457 to reduce the potential
of dead lock with following example:
时间点 | create table线程1 | create table线程2 | table stat线程 | truncate线程
-- | -- | -- | -- | --
t1 | add table1 read lock | | |
t2 | | add table2 read lock | |
t3 | | | try add table1 write lock，blocked |
t4 | | | | try add table2 write lock，blocked
t5 | try add table2 read lock，Due to a thread attempting to add the
table2 write lock earlier, it was forced to queue up | | |
t6 | | try add table1 read lock，Due to a thread attempting to add the
table1 write lock earlier, it was forced to queue up | |
This deadlock situation usually occurs when it is a fair lock. So we
should optimize the read lock.
This PR places the updates of table statistics(such as data size/replica
count) in the Table Stats Thread to be done on a scheduled basis, so
that there is no need to add a read lock when obtaining table
statistics.
<!--Describe your changes.-->
---
.../org/apache/doris/analysis/ShowDataStmt.java | 9 +--
.../java/org/apache/doris/catalog/Database.java | 52 +++++++-------
.../java/org/apache/doris/catalog/OlapTable.java | 79 ++++++++++++++++------
.../org/apache/doris/catalog/TabletStatMgr.java | 14 ++++
4 files changed, 98 insertions(+), 56 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
index f18a86b5cce..bca5cb07fc1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
@@ -151,13 +151,8 @@ public class ShowDataStmt extends ShowStmt {
OlapTable olapTable = (OlapTable) table;
long tableSize = 0;
long replicaCount = 0;
- olapTable.readLock();
- try {
- tableSize = olapTable.getDataSize();
- replicaCount = olapTable.getReplicaCount();
- } finally {
- olapTable.readUnlock();
- }
+ tableSize = olapTable.getDataSize();
+ replicaCount = olapTable.getReplicaCount();
//|TableName|Size|ReplicaCount|
List<Object> row = Arrays.asList(table.getName(),
tableSize, replicaCount);
totalRowsObject.add(row);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index cf7ac4e4adb..bb2b4eb7f19 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -231,48 +231,42 @@ public class Database extends MetaObject implements
Writable, DatabaseIf<Table>
public long getUsedDataQuotaWithLock() {
long usedDataQuota = 0;
+ List<Table> tables = new ArrayList<>();
readLock();
try {
- for (Table table : this.idToTable.values()) {
- if (table.getType() != TableType.OLAP) {
- continue;
- }
-
- OlapTable olapTable = (OlapTable) table;
- olapTable.readLock();
- try {
- usedDataQuota = usedDataQuota + olapTable.getDataSize();
- } finally {
- olapTable.readUnlock();
- }
- }
- return usedDataQuota;
+ tables.addAll(this.idToTable.values());
} finally {
readUnlock();
}
+ for (Table table : tables) {
+ if (table.getType() != TableType.OLAP) {
+ continue;
+ }
+
+ OlapTable olapTable = (OlapTable) table;
+ usedDataQuota = usedDataQuota + olapTable.getDataSize();
+ }
+ return usedDataQuota;
}
public long getReplicaCountWithLock() {
+ long usedReplicaCount = 0;
+ List<Table> tables = new ArrayList<>();
readLock();
try {
- long usedReplicaCount = 0;
- for (Table table : this.idToTable.values()) {
- if (table.getType() != TableType.OLAP) {
- continue;
- }
-
- OlapTable olapTable = (OlapTable) table;
- olapTable.readLock();
- try {
- usedReplicaCount = usedReplicaCount +
olapTable.getReplicaCount();
- } finally {
- olapTable.readUnlock();
- }
- }
- return usedReplicaCount;
+ tables.addAll(this.idToTable.values());
} finally {
readUnlock();
}
+ for (Table table : tables) {
+ if (table.getType() != TableType.OLAP) {
+ continue;
+ }
+
+ OlapTable olapTable = (OlapTable) table;
+ usedReplicaCount = usedReplicaCount + olapTable.getReplicaCount();
+ }
+ return usedReplicaCount;
}
public long getReplicaQuotaLeftWithLock() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 5c9ca6184d9..57f42d2c882 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -86,6 +86,8 @@ import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
+import lombok.Getter;
+
/**
* Internal representation of tableFamilyGroup-related metadata. A
OlaptableFamilyGroup contains several tableFamily.
* Note: when you add a new olap table property, you should modify
TableProperty class
@@ -148,6 +150,8 @@ public class OlapTable extends Table {
private TableProperty tableProperty;
+ private volatile Statistics statistics = new Statistics();
+
public OlapTable() {
// for persist
super(TableType.OLAP);
@@ -1393,26 +1397,6 @@ public class OlapTable extends Table {
return oldPartition;
}
- public long getDataSize(boolean singleReplica) {
- long dataSize = 0;
- for (Partition partition : getAllPartitions()) {
- dataSize += partition.getDataSize(singleReplica);
- }
- return dataSize;
- }
-
- public long getDataSize() {
- return getDataSize(false);
- }
-
- public long getReplicaCount() {
- long replicaCount = 0;
- for (Partition partition : getAllPartitions()) {
- replicaCount += partition.getReplicaCount();
- }
- return replicaCount;
- }
-
public void checkStableAndNormal(String clusterName) throws DdlException {
if (state != OlapTableState.NORMAL) {
throw new DdlException("Table[" + name + "]'s state is not NORMAL.
"
@@ -1995,4 +1979,59 @@ public class OlapTable extends Table {
|| (getKeysType() == KeysType.UNIQUE_KEYS
&& getEnableUniqueKeyMergeOnWrite());
}
+
+ public void setStatistics(Statistics statistics) {
+ this.statistics = statistics;
+ }
+
+ public static class Statistics {
+ @Getter
+ private String dbName;
+ @Getter
+ private String tableName;
+
+ @Getter
+ private Long dataSize; // single replica data size
+ @Getter
+ private Long totalReplicaDataSize;
+
+ @Getter
+ private Long replicaCount;
+
+ public Statistics() {
+ this.dbName = null;
+ this.tableName = null;
+
+ this.dataSize = 0L;
+ this.totalReplicaDataSize = 0L;
+
+ this.replicaCount = 0L;
+ }
+
+ public Statistics(String dbName, String tableName,
+ Long dataSize, Long totalReplicaDataSize, Long
replicaCount) {
+ this.dbName = dbName;
+ this.tableName = tableName;
+
+ this.dataSize = dataSize;
+ this.totalReplicaDataSize = totalReplicaDataSize;
+
+ this.replicaCount = replicaCount;
+ }
+ }
+
+ public long getDataSize() {
+ return getDataSize(false);
+ }
+
+ public long getDataSize(boolean singleReplica) {
+ if (singleReplica) {
+ return statistics.getDataSize();
+ }
+ return statistics.getTotalReplicaDataSize();
+ }
+
+ public long getReplicaCount() {
+ return statistics.getReplicaCount();
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index c541ee5e12b..c9cafac66ae 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -94,6 +94,9 @@ public class TabletStatMgr extends MasterDaemon {
continue;
}
OlapTable olapTable = (OlapTable) table;
+ long tableDataSize = 0L;
+ long tableTotalReplicaDataSize = 0L;
+ long tableReplicaCount = 0L;
if (!table.writeLockIfExist()) {
continue;
}
@@ -103,18 +106,29 @@ public class TabletStatMgr extends MasterDaemon {
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
long indexRowCount = 0L;
for (Tablet tablet : index.getTablets()) {
+ long tabletDataSize = 0L;
long tabletRowCount = 0L;
for (Replica replica : tablet.getReplicas()) {
if (replica.checkVersionCatchUp(version,
false)
&& replica.getRowCount() >
tabletRowCount) {
tabletRowCount = replica.getRowCount();
}
+
+ if (replica.getDataSize() >
tabletDataSize) {
+ tabletDataSize = replica.getDataSize();
+ }
+ tableTotalReplicaDataSize +=
replica.getDataSize();
+ tableReplicaCount++;
}
+ tableDataSize += tabletDataSize;
indexRowCount += tabletRowCount;
} // end for tablets
index.setRowCount(indexRowCount);
} // end for indices
} // end for partitions
+
+ olapTable.setStatistics(new
OlapTable.Statistics(db.getFullName(), table.getName(),
+ tableDataSize, tableTotalReplicaDataSize,
tableReplicaCount));
LOG.debug("finished to set row num for table: {} in
database: {}",
table.getName(), db.getFullName());
} finally {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
