This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new ced6ec07326 [fix](restore) Set the storage medium of the bind replicas
(#40921)
ced6ec07326 is described below
commit ced6ec07326c7b267a9e7d738ffbd731057e9f8b
Author: walter <w41te...@gmail.com>
AuthorDate: Thu Sep 19 15:26:16 2024 +0800
[fix](restore) Set the storage medium of the bind replicas (#40921)
Since the replicas are bound to the same disk, the storage medium must
be the same to avoid media migration.
---
.../java/org/apache/doris/backup/RestoreJob.java | 58 +++++++++++++++++-----
1 file changed, 45 insertions(+), 13 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index 0098dfb3b87..d5d2754fa92 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -652,8 +652,8 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
}
}
- // the new tablets -> { local tablet, schema hash }, used in atomic
restore.
- Map<Long, Pair<Long, Integer>> tabletBases = new HashMap<>();
+ // the new tablets -> { local tablet, schema hash, storage medium },
used in atomic restore.
+ Map<Long, TabletRef> tabletBases = new HashMap<>();
// Check and prepare meta objects.
AgentBatchTask batchTask = new AgentBatchTask();
@@ -1037,14 +1037,30 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
private Status bindLocalAndRemoteOlapTableReplicas(
OlapTable localOlapTbl, OlapTable remoteOlapTbl,
- Map<Long, Pair<Long, Integer>> tabletBases) {
+ Map<Long, TabletRef> tabletBases) {
localOlapTbl.readLock();
try {
+ // The storage medium of the remote olap table's storage is HDD,
because we want to
+ // restore the tables in another cluster might without SSD.
+ //
+ // Keep the storage medium of the new olap table the same as the
old one, so that
+ // the replicas in the new olap table will not be migrated to
other storage mediums.
+ remoteOlapTbl.setStorageMedium(localOlapTbl.getStorageMedium());
for (Partition partition : remoteOlapTbl.getPartitions()) {
Partition localPartition =
localOlapTbl.getPartition(partition.getName());
if (localPartition == null) {
continue;
}
+ // Since the replicas are bound to the same disk, the storage
medium must be the same
+ // to avoid media migration.
+ TStorageMedium storageMedium = localOlapTbl.getPartitionInfo()
+
.getDataProperty(localPartition.getId()).getStorageMedium();
+
remoteOlapTbl.getPartitionInfo().getDataProperty(partition.getId())
+ .setStorageMedium(storageMedium);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("bind local partition {} and remote partition {}
with same storage medium {}, name: {}",
+ localPartition.getId(), partition.getId(),
storageMedium, partition.getName());
+ }
for (MaterializedIndex index :
partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
String indexName =
remoteOlapTbl.getIndexNameById(index.getId());
Long localIndexId =
localOlapTbl.getIndexIdByName(indexName);
@@ -1092,7 +1108,8 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
localOlapTbl.getName());
}
}
- tabletBases.put(remoteTablet.getId(),
Pair.of(localTablet.getId(), schemaHash));
+ tabletBases.put(remoteTablet.getId(),
+ new TabletRef(localTablet.getId(), schemaHash,
storageMedium));
}
}
}
@@ -1223,7 +1240,7 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
}
private void createReplicas(Database db, AgentBatchTask batchTask,
OlapTable localTbl, Partition restorePart,
- Map<Long, Pair<Long, Integer>> tabletBases) {
+ Map<Long, TabletRef> tabletBases) {
Set<String> bfColumns = localTbl.getCopiedBfColumns();
double bfFpp = localTbl.getBfFpp();
@@ -1245,8 +1262,12 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
clusterKeyIndexes =
OlapTable.getClusterKeyIndexes(indexMeta.getSchema());
}
for (Tablet restoreTablet : restoredIdx.getTablets()) {
+ TabletRef baseTabletRef = tabletBases == null ? null :
tabletBases.get(restoreTablet.getId());
+ // All restored replicas will be saved to HDD by default.
+ TStorageMedium storageMedium = baseTabletRef == null
+ ? TStorageMedium.HDD : baseTabletRef.storageMedium;
TabletMeta tabletMeta = new TabletMeta(db.getId(),
localTbl.getId(), restorePart.getId(),
- restoredIdx.getId(), indexMeta.getSchemaHash(),
TStorageMedium.HDD);
+ restoredIdx.getId(), indexMeta.getSchemaHash(),
storageMedium);
Env.getCurrentInvertedIndex().addTablet(restoreTablet.getId(),
tabletMeta);
for (Replica restoreReplica : restoreTablet.getReplicas()) {
Env.getCurrentInvertedIndex().addReplica(restoreTablet.getId(), restoreReplica);
@@ -1255,7 +1276,7 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
restoreTablet.getId(), restoreReplica.getId(),
indexMeta.getShortKeyColumnCount(),
indexMeta.getSchemaHash(),
restoreReplica.getVersion(),
indexMeta.getKeysType(), TStorageType.COLUMN,
- TStorageMedium.HDD /* all restored replicas will
be saved to HDD */,
+ storageMedium,
indexMeta.getSchema(), bfColumns, bfFpp, null,
indexes,
localTbl.isInMemory(),
@@ -1280,12 +1301,11 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
localTbl.variantEnableFlattenNested());
task.setInvertedIndexFileStorageFormat(localTbl.getInvertedIndexFileStorageFormat());
task.setInRestoreMode(true);
- if (tabletBases != null &&
tabletBases.containsKey(restoreTablet.getId())) {
+ if (baseTabletRef != null) {
// ensure this replica is bound to the same backend
disk as the origin table's replica.
- Pair<Long, Integer> baseTablet =
tabletBases.get(restoreTablet.getId());
- task.setBaseTablet(baseTablet.first,
baseTablet.second);
+ task.setBaseTablet(baseTabletRef.tabletId,
baseTabletRef.schemaHash);
LOG.info("set base tablet {} for replica {} in restore
job {}, tablet id={}",
- baseTablet.first, restoreReplica.getId(),
jobId, restoreTablet.getId());
+ baseTabletRef.tabletId,
restoreReplica.getId(), jobId, restoreTablet.getId());
}
if (!CollectionUtils.isEmpty(clusterKeyIndexes)) {
task.setClusterKeyIndexes(clusterKeyIndexes);
@@ -1379,7 +1399,7 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
}
private void genFileMapping(OlapTable localTbl, Partition localPartition,
Long remoteTblId,
- BackupPartitionInfo backupPartInfo, boolean overwrite, Map<Long,
Pair<Long, Integer>> tabletBases) {
+ BackupPartitionInfo backupPartInfo, boolean overwrite, Map<Long,
TabletRef> tabletBases) {
for (MaterializedIndex localIdx :
localPartition.getMaterializedIndices(IndexExtState.VISIBLE)) {
if (LOG.isDebugEnabled()) {
LOG.debug("get index id: {}, index name: {}", localIdx.getId(),
@@ -1396,7 +1416,7 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
for (Replica localReplica : localTablet.getReplicas()) {
long refTabletId = -1L;
if (tabletBases != null &&
tabletBases.containsKey(localTablet.getId())) {
- refTabletId =
tabletBases.get(localTablet.getId()).first;
+ refTabletId =
tabletBases.get(localTablet.getId()).tabletId;
if (LOG.isDebugEnabled()) {
LOG.debug("restored tablet {} is based on exists
tablet {}",
localTablet.getId(), refTabletId);
@@ -2565,4 +2585,16 @@ public class RestoreJob extends AbstractJob implements
GsonPostProcessable {
public static String tableAliasWithAtomicRestore(String tableName) {
return ATOMIC_RESTORE_TABLE_PREFIX + tableName;
}
+
+ private static class TabletRef {
+ public long tabletId;
+ public int schemaHash;
+ public TStorageMedium storageMedium;
+
+ TabletRef(long tabletId, int schemaHash, TStorageMedium storageMedium)
{
+ this.tabletId = tabletId;
+ this.schemaHash = schemaHash;
+ this.storageMedium = storageMedium;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
