This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push:
new b6a18031270 [improve](backup) Add config ignore_backup_tmp_partitions
#45240 (#45331)
b6a18031270 is described below
commit b6a18031270366866a60ed5efff482d47cf0e1d3
Author: walter <maoch...@selectdb.com>
AuthorDate: Tue Dec 17 14:31:06 2024 +0800
[improve](backup) Add config ignore_backup_tmp_partitions #45240 (#45331)
cherry pick from #45240
---
.../main/java/org/apache/doris/common/Config.java | 9 ++++++
.../org/apache/doris/backup/BackupHandler.java | 4 +--
.../java/org/apache/doris/backup/BackupJob.java | 16 ++++++----
.../java/org/apache/doris/catalog/OlapTable.java | 8 +++++
...est_backup_restore_backup_temp_partition.groovy | 34 ++++++++++++++++++++++
5 files changed, 63 insertions(+), 8 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index ec09c850805..e0f0b0064eb 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1589,6 +1589,15 @@ public class Config extends ConfigBase {
@ConfField(mutable = true, masterOnly = true)
public static boolean ignore_backup_not_support_table_type = false;
+ /**
+ * whether to ignore temp partitions when backup, and not report exception.
+ */
+ @ConfField(mutable = true, masterOnly = true, description = {
+ "是否忽略备份临时分区，不报异常",
+ "Whether to ignore temp partitions when backup, and not report
exception."
+ })
+ public static boolean ignore_backup_tmp_partitions = false;
+
/**
* A internal config, to control the update interval of backup handler.
Only used to speed up tests.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 4749bf7f619..34e127d137a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -427,14 +427,14 @@ public class BackupHandler extends MasterDaemon
implements Writable {
OlapTable olapTbl = (OlapTable) tbl;
tbl.readLock();
try {
- if (olapTbl.existTempPartitions()) {
+ if (!Config.ignore_backup_tmp_partitions &&
olapTbl.existTempPartitions()) {
ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Do not support backup table " + olapTbl.getName()
+ " with temp partitions");
}
PartitionNames partitionNames = tblRef.getPartitionNames();
if (partitionNames != null) {
- if (partitionNames.isTemp()) {
+ if (!Config.ignore_backup_tmp_partitions &&
partitionNames.isTemp()) {
ErrorReport.reportDdlException(ErrorCode.ERR_COMMON_ERROR,
"Do not support backup temp partitions in
table " + tblRef.getName());
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 05349cbcc28..e1672dad922 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -597,11 +597,15 @@ public class BackupJob extends AbstractJob {
// check backup table again
if (backupTableRef.getPartitionNames() != null) {
for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
+ Partition partition = olapTable.getPartition(partName, false);
// exclude tmp partitions
if (partition == null) {
- status = new Status(ErrCode.NOT_FOUND, "partition " +
partName
- + " does not exist in table" +
backupTableRef.getName().getTbl());
- return status;
+ if (olapTable.getPartition(partName, true) != null) {
+ return new Status(ErrCode.NOT_FOUND, "backup tmp
partition " + partName
+ + " in table " +
backupTableRef.getName().getTbl() + " is not supported");
+ } else {
+ return new Status(ErrCode.NOT_FOUND, "partition " +
partName
+ + " does not exist in table " +
backupTableRef.getName().getTbl());
+ }
}
}
}
@@ -609,10 +613,10 @@ public class BackupJob extends AbstractJob {
// create snapshot tasks
List<Partition> partitions = Lists.newArrayList();
if (backupTableRef.getPartitionNames() == null) {
- partitions.addAll(olapTable.getPartitions());
+ partitions.addAll(olapTable.getPartitions()); // no temp
partitions in OlapTable.getPartitions()
} else {
for (String partName :
backupTableRef.getPartitionNames().getPartitionNames()) {
- Partition partition = olapTable.getPartition(partName);
+ Partition partition = olapTable.getPartition(partName, false);
// exclude tmp partitions
partitions.add(partition);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index 463f335d878..d64b8febc7d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -1935,6 +1935,14 @@ public class OlapTable extends Table implements
MTMVRelatedTableIf {
}
}
+ if (isForBackup) {
+ // drop all tmp partitions in copied table
+ for (Partition partition :
copied.tempPartitions.getAllPartitions()) {
+ copied.partitionInfo.dropPartition(partition.getId());
+ }
+ copied.tempPartitions = new TempPartitions();
+ }
+
if (reservedPartitions == null || reservedPartitions.isEmpty()) {
// reserve all
return copied;
diff --git
a/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
b/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
index 61514bedd88..8f06ff4eb8b 100644
---
a/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
+++
b/regression-test/suites/backup_restore/test_backup_restore_backup_temp_partition.groovy
@@ -62,6 +62,7 @@ suite("test_backup_restore_backup_temp_partition",
"backup_restore") {
ALTER TABLE ${dbName}.${tableName} ADD TEMPORARY PARTITION tp1 VALUES
LESS THAN ("70")
"""
+ sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'false')"
test {
sql """
BACKUP SNAPSHOT ${dbName}.${snapshotName}
@@ -71,6 +72,39 @@ suite("test_backup_restore_backup_temp_partition",
"backup_restore") {
exception "Do not support backup table ${tableName} with temp
partitions"
}
+ // ignore the tmp partitions
+ sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'true')"
+ sql """
+ BACKUP SNAPSHOT ${dbName}.${snapshotName}
+ TO `${repoName}`
+ ON (${tableName})
+ """
+
+ syncer.waitSnapshotFinish(dbName)
+ def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName)
+ assertTrue(snapshot != null)
+
+ // The restored table has no tmp partitions
+ sql "DROP TABLE IF EXISTS ${dbName}.${tableName}"
+
+ sql """
+ RESTORE SNAPSHOT ${dbName}.${snapshotName}
+ FROM `${repoName}`
+ ON (
+ `${tableName}` PARTITION (p1, p2, p3)
+ )
+ PROPERTIES
+ (
+ "backup_timestamp" = "${snapshot}",
+ "reserve_replica" = "true"
+ )
+ """
+
+ syncer.waitAllRestoreFinish(dbName)
+ def res = sql "SHOW TEMPORARY PARTITIONS FROM ${dbName}.${tableName}"
+ assertTrue(res.size() == 0);
+
+ sql "ADMIN SET FRONTEND CONFIG ('ignore_backup_tmp_partitions' = 'false')"
sql "DROP TABLE ${dbName}.${tableName} FORCE"
sql "DROP DATABASE ${dbName} FORCE"
sql "DROP REPOSITORY `${repoName}`"
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
