This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new f13363a4155 [fix](restore) Reset next version for restored partitions
(#38344)
f13363a4155 is described below
commit f13363a41554d78ad59fe5b09a0187ddcc2bc8ef
Author: walter <w41te...@gmail.com>
AuthorDate: Thu Jul 25 20:50:01 2024 +0800
[fix](restore) Reset next version for restored partitions (#38344)
Cherry-pick #38321
---
.../java/org/apache/doris/backup/RestoreJob.java | 10 ++++++--
.../java/org/apache/doris/catalog/OlapTable.java | 4 ++++
.../org/apache/doris/backup/RestoreJobTest.java | 27 ++++++++++++++++++++++
3 files changed, 39 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 4224dc6c3e7..1b6b4b47072 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -82,6 +82,7 @@ import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.thrift.TStorageType;
import org.apache.doris.thrift.TTaskType;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.ArrayListMultimap;
@@ -1081,8 +1082,9 @@ public class RestoreJob extends AbstractJob {
// reset remote partition.
// reset all id in remote partition, but DO NOT modify any exist catalog
objects.
- private Partition resetPartitionForRestore(OlapTable localTbl, OlapTable
remoteTbl, String partName,
- String clusterName,
ReplicaAllocation replicaAlloc) {
+ @VisibleForTesting
+ protected Partition resetPartitionForRestore(OlapTable localTbl, OlapTable
remoteTbl, String partName,
+ String clusterName,
ReplicaAllocation replicaAlloc) {
Preconditions.checkState(localTbl.getPartition(partName) == null);
Partition remotePart = remoteTbl.getPartition(partName);
Preconditions.checkNotNull(remotePart);
@@ -1092,6 +1094,7 @@ public class RestoreJob extends AbstractJob {
// generate new partition id
long newPartId = env.getNextId();
+ long oldPartId = remotePart.getId();
remotePart.setIdForRestore(newPartId);
// indexes
@@ -1111,6 +1114,9 @@ public class RestoreJob extends AbstractJob {
// save version info for creating replicas
long visibleVersion = remotePart.getVisibleVersion();
+ remotePart.setNextVersion(visibleVersion + 1);
+ LOG.info("reset partition {} for restore, visible version: {}, old
partition id: {}",
+ newPartId, visibleVersion, oldPartId);
// tablets
Map<Tag, Integer> nextIndexs = Maps.newHashMap();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
index c3070d903f9..d357b9f93d6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -323,6 +323,10 @@ public class OlapTable extends Table {
}
}
+ public void setPartitionInfo(PartitionInfo info) {
+ partitionInfo = info;
+ }
+
public boolean hasMaterializedIndex(String indexName) {
return indexNameToId.containsKey(indexName);
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
index 7e55f94f8d9..5f426aa3311 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/backup/RestoreJobTest.java
@@ -24,10 +24,13 @@ import
org.apache.doris.backup.BackupJobInfo.BackupPartitionInfo;
import org.apache.doris.backup.BackupJobInfo.BackupTabletInfo;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.Env;
+import org.apache.doris.catalog.HashDistributionInfo;
import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
+import org.apache.doris.catalog.PartitionInfo;
+import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
@@ -51,6 +54,7 @@ import mockit.Injectable;
import mockit.Mock;
import mockit.MockUp;
import mockit.Mocked;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -276,4 +280,27 @@ public class RestoreJobTest {
System.out.println("tbl signature: " +
tbl.getSignature(BackupHandler.SIGNATURE_VERSION, partNames));
}
+ @Test
+ public void testResetPartitionVisibleAndNextVersionForRestore() throws
Exception {
+ long visibleVersion = 1234;
+ long remotePartId = 123;
+ String partName = "p20240723";
+ MaterializedIndex index = new MaterializedIndex();
+ Partition remotePart = new Partition(remotePartId, partName, index,
new HashDistributionInfo());
+ remotePart.setVisibleVersionAndTime(visibleVersion, 0);
+ remotePart.setNextVersion(visibleVersion + 10);
+
+ OlapTable localTbl = new OlapTable();
+ localTbl.setPartitionInfo(new PartitionInfo(PartitionType.RANGE));
+ OlapTable remoteTbl = new OlapTable();
+ remoteTbl.addPartition(remotePart);
+ remoteTbl.setPartitionInfo(new PartitionInfo(PartitionType.RANGE));
+
+ ReplicaAllocation alloc = new ReplicaAllocation();
+ job.resetPartitionForRestore(localTbl, remoteTbl, partName, "", alloc);
+
+ Partition localPart = remoteTbl.getPartition(partName);
+ Assert.assertEquals(localPart.getVisibleVersion(), visibleVersion);
+ Assert.assertEquals(localPart.getNextVersion(), visibleVersion + 1);
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
