This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new 1617866a2c4 [fix](load) fix commit txn timeout when loading to table
with many tablet (#40031) (#40140)
1617866a2c4 is described below
commit 1617866a2c48ed9391a7c91b9db10044c8777eb1
Author: zclllhhjj <zhaochan...@selectdb.com>
AuthorDate: Wed Sep 11 09:28:16 2024 +0800
[fix](load) fix commit txn timeout when loading to table with many tablet
(#40031) (#40140)
---
.../org/apache/doris/load/BrokerFileGroup.java | 17 +++++---
.../doris/transaction/DatabaseTransactionMgr.java | 50 ++++++++++++++--------
2 files changed, 43 insertions(+), 24 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
index 35bd0b8db7d..01e46012e66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
@@ -191,13 +191,16 @@ public class BrokerFileGroup implements Writable {
}
}
- boolean isPartitionRestoring =
olapTable.getPartitions().stream().anyMatch(
- partition -> partition.getState() == PartitionState.RESTORE
- );
-
- // restore table
- if (!isPartitionRestoring && olapTable.getState() ==
OlapTableState.RESTORE) {
- throw new DdlException("Table [" + olapTable.getName() + "] is
under restore");
+ // only do check when here's restore on this table now
+ if (olapTable.getState() == OlapTableState.RESTORE) {
+ boolean hasPartitionRestoring =
olapTable.getPartitions().stream()
+ .anyMatch(partition -> partition.getState() ==
PartitionState.RESTORE);
+ // tbl RESTORE && all partition NOT RESTORE -> whole table
restore
+ // tbl RESTORE && some partition RESTORE -> just partitions
restore, NOT WHOLE TABLE
+ // so check wether the whole table restore here
+ if (!hasPartitionRestoring) {
+ throw new DdlException("Table [" + olapTable.getName() +
"] is under restore");
+ }
}
if (olapTable.getKeysType() != KeysType.AGG_KEYS &&
dataDescription.isNegative()) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index b83d4c9c98b..cf4bed28e5b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -462,7 +462,9 @@ public class DatabaseTransactionMgr {
List<Long> tabletIds = tabletCommitInfos.stream()
.map(TabletCommitInfo::getTabletId).collect(Collectors.toList());
List<TabletMeta> tabletMetaList =
tabletInvertedIndex.getTabletMetaList(tabletIds);
+ HashMap<Long, Boolean> tableIdtoRestoring = new HashMap<>();
for (int i = 0; i < tabletMetaList.size(); i++) {
+ // get partition and table of this tablet
TabletMeta tabletMeta = tabletMetaList.get(i);
if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
continue;
@@ -471,29 +473,43 @@ public class DatabaseTransactionMgr {
long tableId = tabletMeta.getTableId();
OlapTable tbl = (OlapTable) idToTable.get(tableId);
if (tbl == null) {
- // this can happen when tableId == -1 (tablet being dropping)
- // or table really not exist.
+ // this can happen when tableId == -1 (tablet being dropping)
or table really not exist.
continue;
}
+ // check relative partition restore here
long partitionId = tabletMeta.getPartitionId();
- Partition partition = tbl.getPartition(partitionId);
- if (partition == null) {
- // this can happen when partitionId == -1 (tablet being
dropping)
- // or partition really not exist.
+ if (tbl.getPartition(partitionId) == null) {
+ // this can happen when partitionId == -1 (tablet being
dropping) or partition really not exist.
continue;
- } else if (partition.getState() == PartitionState.RESTORE) {
- // partition which need load data
+ }
+ if (tbl.getPartition(partitionId).getState() ==
PartitionState.RESTORE) {
+ // partition in restore process which can not load data
throw new LoadException("Table [" + tbl.getName() + "],
Partition ["
- + partition.getName() + "] is in restore process. Can
not load into it");
- }
- boolean isPartitionRestoring =
tbl.getPartitions().stream().anyMatch(
- curPartition -> curPartition.getState() ==
PartitionState.RESTORE
- );
- // restore table
- if (!isPartitionRestoring && tbl.getState() ==
OlapTableState.RESTORE) {
- throw new LoadException("Table " + tbl.getName() + " is in
restore process. "
- + "Can not load into it");
+ + tbl.getPartition(partitionId).getName() + "] is in
restore process. Can not load into it");
+ }
+
+ // only do check when here's restore on this table now
+ if (tbl.getState() == OlapTableState.RESTORE) {
+ boolean hasPartitionRestoring = false;
+ if (tableIdtoRestoring.containsKey(tableId)) {
+ hasPartitionRestoring = tableIdtoRestoring.get(tableId);
+ } else {
+ for (Partition partition : tbl.getPartitions()) {
+ if (partition.getState() == PartitionState.RESTORE) {
+ hasPartitionRestoring = true;
+ break;
+ }
+ }
+ tableIdtoRestoring.put(tableId, hasPartitionRestoring);
+ }
+ // tbl RESTORE && all partition NOT RESTORE -> whole table
restore
+ // tbl RESTORE && some partition RESTORE -> just partitions
restore, NOT WHOLE TABLE
+ // so check wether the whole table restore here
+ if (!hasPartitionRestoring) {
+ throw new LoadException(
+ "Table " + tbl.getName() + " is in restore
process. " + "Can not load into it");
+ }
}
if (!tableToPartition.containsKey(tableId)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
