This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 1c43ee70c29c59b0519e495aee409721e30b8728
Author: shouchengShen <50309488+johnny...@users.noreply.github.com>
AuthorDate: Mon Aug 26 15:22:06 2024 +0800
[enhancement](Load)allow load data to the other partitions when some
partitions are restoring (#39595)
If broker load or stream load task execute in one table that is
restoring data, load task will failed with Exception.
Exception info :"Table [xxx] is under restore" or "Table [xxx] is in
restore process, can't load into it".
But mostly restoreJob only effects some partitions in this table, not
all of them, so that the other partitions still need to load data
successfully.
To achieve this goal, before checking olap table state, check partition
state first.
ps: set restore status for partitions in this
pr:https://github.com/apache/doris/pull/8245
## test case for this pr
### restore tbl's partition p202408
$ RESTORE SNAPSHOT db.tbl_p202408_test
FROM repo
ON(
`tbl` PARTITION (p202408)
)
PROPERTIES(
"backup_timestamp"="2024-08-22-20-32-37",
"replication_num" = "1"
);
### check restore job state\G
$ SHOW RESTORE\G
*************************** 1. row ***************************
JobId: 21741
Label: tbl_p202408_test
Timestamp: 2024-08-22-20-32-37
State: DOWNLOADING
RestoreObjs: {
"name": "tbl_p202408_test",
"database": "db",
"olap_table_list": [
{
"name": "tbl",
"partition_names": ["p202408"]
}
]
### load to partition p202408, failed with exception
curl --location-trusted -u root:"" \
> -H "label:tbl_test_load_19" \
> -H "timeout:300" \
> -H "format: parquet" \
> -T data_for_p202408.parquet \
> -XPUT http://fe_ip:8030/api/db/tbl/_stream_load
{
"TxnId": 3042,
"Label": "tbl_test_load_19",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Fail",
"Message": "[ANALYSIS_ERROR]TStatus: errCode = 2, detailMessage = Table
[zt_order_detail_v3], Partition [p202408] is in restore process. Can not
load into it.etc.",
"NumberTotalRows": 682,
"NumberLoadedRows": 682,
"NumberFilteredRows": 0,
"NumberUnselectedRows": 0,
"LoadBytes": 82025,
"LoadTimeMs": 48,
"BeginTxnTimeMs": 0,
"StreamLoadPutTimeMs": 7,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 38,
"CommitAndPublishTimeMs": 0
}
### load to partition p202408, successfully
$ curl --location-trusted -u root:"" \
> -H "timeout:300" \
> -H "format: json" \
> -H "read_json_by_line:true" \
> -T data_for_p202407.json \
> -XPUT http://fe_ip:8030/api/db/tbl/_stream_load
{
"TxnId": 3043,
"Label": "2f2dae38-a495-4c22-9492-419ea70b724e",
"Comment": "",
"TwoPhaseCommit": "false",
"Status": "Success",
"Message": "OK",
"NumberTotalRows": 1,
"NumberLoadedRows": 1,
"NumberFilteredRows": 0,
"NumberUnselectedRows": 0,
"LoadBytes": 1128,
"LoadTimeMs": 51,
"BeginTxnTimeMs": 0,
"StreamLoadPutTimeMs": 6,
"ReadDataTimeMs": 0,
"WriteDataTimeMs": 30,
"CommitAndPublishTimeMs": 13
}
Co-authored-by: shenshoucheng <shenshouch...@jd.com>
---
.../java/org/apache/doris/load/BrokerFileGroup.java | 12 +++++++++++-
.../doris/transaction/DatabaseTransactionMgr.java | 20 +++++++++++++++-----
2 files changed, 26 insertions(+), 6 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
index 8f383de0d92..d182e1d08d4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
@@ -29,6 +29,7 @@ import org.apache.doris.catalog.KeysType;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.Table;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
@@ -142,11 +143,20 @@ public class BrokerFileGroup implements Writable {
throw new DdlException("Unknown partition '" + pName
+ "' in table '" + olapTable.getName() + "'");
}
+ // partition which need load data
+ if (partition.getState() == PartitionState.RESTORE) {
+ throw new DdlException("Table [" + olapTable.getName()
+ + "], Partition[" + partition.getName() + "]
is under restore");
+ }
partitionIds.add(partition.getId());
}
}
- if (olapTable.getState() == OlapTableState.RESTORE) {
+ boolean isPartitionRestoring =
olapTable.getPartitions().stream().anyMatch(
+ partition -> partition.getState() == PartitionState.RESTORE
+ );
+ // restore table
+ if (!isPartitionRestoring && olapTable.getState() ==
OlapTableState.RESTORE) {
throw new DdlException("Table [" + olapTable.getName() + "] is
under restore");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index d3ab8ef9bc6..7752a81e850 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -23,7 +23,9 @@ import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.OlapTable;
+import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
@@ -521,16 +523,24 @@ public class DatabaseTransactionMgr {
continue;
}
- if (tbl.getState() == OlapTable.OlapTableState.RESTORE) {
- throw new LoadException("Table " + tbl.getName() + " is in
restore process. "
- + "Can not load into it");
- }
-
long partitionId = tabletMeta.getPartitionId();
if (tbl.getPartition(partitionId) == null) {
// this can happen when partitionId == -1 (tablet being
dropping)
// or partition really not exist.
continue;
+ } else if (tbl.getPartition(partitionId).getState() ==
PartitionState.RESTORE) {
+ // partition in restore process which can not load data
+ throw new LoadException("Table [" + tbl.getName() + "],
Partition ["
+ + tbl.getPartition(partitionId).getName() + "] is in
restore process. Can not load into it");
+ }
+
+ boolean isPartitionRestoring =
tbl.getPartitions().stream().anyMatch(
+ partition -> partition.getState() == PartitionState.RESTORE
+ );
+ // restore table
+ if (!isPartitionRestoring && tbl.getState() ==
OlapTableState.RESTORE) {
+ throw new LoadException("Table " + tbl.getName() + " is in
restore process. "
+ + "Can not load into it");
}
if (!tableToPartition.containsKey(tableId)) {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
