This is an automated email from the ASF dual-hosted git repository.
zhaoc pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6928c72 Optimize the logic for getting TabletMeta from
TabletInvertedIndex to reduce frequency of getting read lock (#3815)
6928c72 is described below
commit 6928c727038bc5449fb700e200c3fbc206edfa1b
Author: caiconghui <55968745+caicong...@users.noreply.github.com>
AuthorDate: Sat Jun 13 12:46:59 2020 +0800
Optimize the logic for getting TabletMeta from TabletInvertedIndex to
reduce frequency of getting read lock (#3815)
This PR is to optimize the logic for getting tabletMeta from
TabletInvertedIndex to reduce frequence of getting read lock
---
.../apache/doris/catalog/TabletInvertedIndex.java | 84 +++-------------------
.../java/org/apache/doris/master/MasterImpl.java | 48 ++++++++-----
.../org/apache/doris/master/ReportHandler.java | 73 ++++++++++++-------
.../java/org/apache/doris/qe/ShowExecutor.java | 11 +--
.../doris/transaction/DatabaseTransactionMgr.java | 18 +++--
.../doris/transaction/PublishVersionDaemon.java | 11 ++-
6 files changed, 116 insertions(+), 129 deletions(-)
diff --git a/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
b/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
index 74d7beb..acd3096 100644
--- a/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
+++ b/fe/src/main/java/org/apache/doris/catalog/TabletInvertedIndex.java
@@ -40,6 +40,7 @@ import com.google.common.collect.Table;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -58,6 +59,9 @@ public class TabletInvertedIndex {
public static final int NOT_EXIST_VALUE = -1;
+ public static final TabletMeta NOT_EXIST_TABLET_META = new
TabletMeta(NOT_EXIST_VALUE, NOT_EXIST_VALUE,
+ NOT_EXIST_VALUE, NOT_EXIST_VALUE, NOT_EXIST_VALUE,
TStorageMedium.HDD);
+
private ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
// tablet id -> tablet meta
@@ -252,43 +256,6 @@ public class TabletInvertedIndex {
tabletMigrationMap.size(), transactionsToClear.size(),
transactionsToPublish.size(), (end - start));
}
- public long getDbId(long tabletId) {
- readLock();
- try {
- if (tabletMetaMap.containsKey(tabletId)) {
- return tabletMetaMap.get(tabletId).getDbId();
- }
- return NOT_EXIST_VALUE;
- } finally {
- readUnlock();
- }
- }
-
- public long getTableId(long tabletId) {
- readLock();
- try {
- if (tabletMetaMap.containsKey(tabletId)) {
- return tabletMetaMap.get(tabletId).getTableId();
- }
- return NOT_EXIST_VALUE;
- } finally {
- readUnlock();
- }
- }
-
- public TabletMeta getTabletMetaByReplica(long replicaId) {
- readLock();
- try {
- Long tabletId = replicaToTabletMap.get(replicaId);
- if (tabletId == null) {
- return null;
- }
- return tabletMetaMap.get(tabletId);
- } finally {
- readUnlock();
- }
- }
-
public Long getTabletIdByReplica(long replicaId) {
readLock();
try {
@@ -298,57 +265,28 @@ public class TabletInvertedIndex {
}
}
- public long getPartitionId(long tabletId) {
- readLock();
- try {
- if (tabletMetaMap.containsKey(tabletId)) {
- return tabletMetaMap.get(tabletId).getPartitionId();
- }
- return NOT_EXIST_VALUE;
- } finally {
- readUnlock();
- }
- }
-
- public long getIndexId(long tabletId) {
+ public TabletMeta getTabletMeta(long tabletId) {
readLock();
try {
- if (tabletMetaMap.containsKey(tabletId)) {
- return tabletMetaMap.get(tabletId).getIndexId();
- }
- return NOT_EXIST_VALUE;
+ return tabletMetaMap.get(tabletId);
} finally {
readUnlock();
}
}
- public int getEffectiveSchemaHash(long tabletId) {
- // always get old schema hash(as effective one)
+ public List<TabletMeta> getTabletMetaList(List<Long> tabletIdList) {
+ List<TabletMeta> tabletMetaList = new ArrayList<>(tabletIdList.size());
readLock();
try {
- if (tabletMetaMap.containsKey(tabletId)) {
- return tabletMetaMap.get(tabletId).getOldSchemaHash();
+ for (Long tabletId : tabletIdList) {
+ tabletMetaList.add(tabletMetaMap.getOrDefault(tabletId,
NOT_EXIST_TABLET_META));
}
- return NOT_EXIST_VALUE;
+ return tabletMetaList;
} finally {
readUnlock();
}
}
- public TabletMeta getTabletMeta(long tabletId) {
- readLock();
- try {
- return tabletMetaMap.get(tabletId);
- } finally {
- readUnlock();
- }
- }
-
- public Set<Long> getTabletBackends(long tabletId) {
- Map<Long, Replica> backendIdToReplica = replicaMetaTable.row(tabletId);
- return backendIdToReplica.keySet();
- }
-
private boolean needSync(Replica replicaInFe, TTabletInfo
backendTabletInfo) {
if (backendTabletInfo.isSetUsed() && !backendTabletInfo.isUsed()) {
// tablet is bad, do not sync
diff --git a/fe/src/main/java/org/apache/doris/master/MasterImpl.java
b/fe/src/main/java/org/apache/doris/master/MasterImpl.java
index 25c3f9f..9d403cd 100644
--- a/fe/src/main/java/org/apache/doris/master/MasterImpl.java
+++ b/fe/src/main/java/org/apache/doris/master/MasterImpl.java
@@ -76,6 +76,7 @@ import org.apache.thrift.TException;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
+import java.util.stream.Collectors;
public class MasterImpl {
private static final Logger LOG = LogManager.getLogger(MasterImpl.class);
@@ -345,6 +346,11 @@ public class MasterImpl {
long reportVersion = request.getReport_version();
Catalog.getCurrentSystemInfo().updateBackendReportVersion(task.getBackendId(),
reportVersion,
task.getDbId());
+
+ List<Long> tabletIds = finishTabletInfos.stream().map(
+ tTabletInfo ->
tTabletInfo.getTablet_id()).collect(Collectors.toList());
+ List<TabletMeta> tabletMetaList =
Catalog.getCurrentInvertedIndex().getTabletMetaList(tabletIds);
+
// handle load job
// TODO yiguolei: why delete should check request version and task
version?
if (pushTask.getPushType() == TPushType.LOAD ||
pushTask.getPushType() == TPushType.LOAD_DELETE) {
@@ -353,11 +359,12 @@ public class MasterImpl {
if (job == null) {
throw new MetaNotFoundException("cannot find load job,
job[" + loadJobId + "]");
}
- for (TTabletInfo tTabletInfo : finishTabletInfos) {
- checkReplica(olapTable, partition, backendId, pushIndexId,
pushTabletId,
- tTabletInfo, pushState);
- Replica replica = findRelatedReplica(olapTable, partition,
- backendId, tTabletInfo);
+
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ checkReplica(finishTabletInfos.get(i), tabletMeta);
+ long tabletId = tabletIds.get(i);
+ Replica replica = findRelatedReplica(olapTable, partition,
backendId, tabletId, tabletMeta.getIndexId());
// if the replica is under schema change, could not find
the replica with aim schema hash
if (replica != null) {
job.addFinishedReplica(replica);
@@ -368,9 +375,11 @@ public class MasterImpl {
if (deleteJob == null) {
throw new MetaNotFoundException("cannot find delete job,
job[" + transactionId + "]");
}
- for (TTabletInfo tTabletInfo : finishTabletInfos) {
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ long tabletId = tabletIds.get(i);
Replica replica = findRelatedReplica(olapTable, partition,
- backendId, tTabletInfo);
+ backendId, tabletId, tabletMeta.getIndexId());
if (replica != null) {
deleteJob.addFinishedReplica(pushTabletId, replica);
pushTask.countDownLatch(backendId, pushTabletId);
@@ -388,8 +397,7 @@ public class MasterImpl {
}
}
- private void checkReplica(OlapTable olapTable, Partition partition, long
backendId,
- long pushIndexId, long pushTabletId, TTabletInfo tTabletInfo,
PartitionState pushState)
+ private void checkReplica(TTabletInfo tTabletInfo, TabletMeta tabletMeta)
throws MetaNotFoundException {
long tabletId = tTabletInfo.getTablet_id();
int schemaHash = tTabletInfo.getSchema_hash();
@@ -398,8 +406,7 @@ public class MasterImpl {
// the check replica will failed
// should use tabletid not pushTabletid because in rollup state, the
push tabletid != tabletid
// and tablet meta will not contain rollupindex's schema hash
- TabletMeta tabletMeta =
Catalog.getCurrentInvertedIndex().getTabletMeta(tabletId);
- if (tabletMeta == null) {
+ if (tabletMeta == null || tabletMeta ==
TabletInvertedIndex.NOT_EXIST_TABLET_META) {
// rollup may be dropped
throw new MetaNotFoundException("tablet " + tabletId + " does not
exist");
}
@@ -411,10 +418,8 @@ public class MasterImpl {
}
private Replica findRelatedReplica(OlapTable olapTable, Partition
partition,
- long backendId, TTabletInfo
tTabletInfo)
+ long backendId, long
tabletId, long indexId)
throws MetaNotFoundException {
- long tabletId = tTabletInfo.getTablet_id();
- long indexId = Catalog.getCurrentInvertedIndex().getIndexId(tabletId);
// both normal index and rollingup index are in inverted index
// this means the index is dropped during load
if (indexId == TabletInvertedIndex.NOT_EXIST_VALUE) {
@@ -529,10 +534,16 @@ public class MasterImpl {
// update replica version and versionHash
List<ReplicaPersistInfo> infos = new
LinkedList<ReplicaPersistInfo>();
- for (TTabletInfo tTabletInfo : finishTabletInfos) {
+ List<Long> tabletIds = finishTabletInfos.stream().map(
+ finishTabletInfo ->
finishTabletInfo.getTablet_id()).collect(Collectors.toList());
+ List<TabletMeta> tabletMetaList =
Catalog.getCurrentInvertedIndex().getTabletMetaList(tabletIds);
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ TTabletInfo tTabletInfo = finishTabletInfos.get(i);
+ long indexId = tabletMeta.getIndexId();
ReplicaPersistInfo info = updateReplicaInfo(olapTable,
partition,
- backendId,
pushIndexId, pushTabletId,
- tTabletInfo,
pushState);
+ backendId, pushIndexId, indexId,
+ tTabletInfo, pushState);
if (info != null) {
infos.add(info);
}
@@ -623,7 +634,7 @@ public class MasterImpl {
}
private ReplicaPersistInfo updateReplicaInfo(OlapTable olapTable,
Partition partition,
- long backendId, long
pushIndexId, long pushTabletId,
+ long backendId, long
pushIndexId, long indexId,
TTabletInfo tTabletInfo,
PartitionState pushState)
throws MetaNotFoundException {
long tabletId = tTabletInfo.getTablet_id();
@@ -633,7 +644,6 @@ public class MasterImpl {
long rowCount = tTabletInfo.getRow_count();
long dataSize = tTabletInfo.getData_size();
- long indexId = Catalog.getCurrentInvertedIndex().getIndexId(tabletId);
if (indexId != pushIndexId) {
// this may be a rollup tablet
if (pushState != PartitionState.ROLLUP && indexId !=
TabletInvertedIndex.NOT_EXIST_VALUE) {
diff --git a/fe/src/main/java/org/apache/doris/master/ReportHandler.java
b/fe/src/main/java/org/apache/doris/master/ReportHandler.java
index 6f13734..18550af 100644
--- a/fe/src/main/java/org/apache/doris/master/ReportHandler.java
+++ b/fe/src/main/java/org/apache/doris/master/ReportHandler.java
@@ -32,6 +32,7 @@ import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.Tablet.TabletStatus;
import org.apache.doris.catalog.TabletInvertedIndex;
+import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.clone.TabletSchedCtx;
import org.apache.doris.common.Config;
import org.apache.doris.common.MetaNotFoundException;
@@ -388,21 +389,26 @@ public class ReportHandler extends Daemon {
List<Long> tabletIds = tabletSyncMap.get(dbId);
LOG.info("before sync tablets in db[{}]. report num: {}.
backend[{}]",
dbId, tabletIds.size(), backendId);
-
- for (Long tabletId : tabletIds) {
- long tableId = invertedIndex.getTableId(tabletId);
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(tabletIds);
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta ==
TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ long tabletId = tabletIds.get(i);
+ long tableId = tabletMeta.getTableId();
OlapTable olapTable = (OlapTable) db.getTable(tableId);
if (olapTable == null) {
continue;
}
- long partitionId = invertedIndex.getPartitionId(tabletId);
+ long partitionId = tabletMeta.getPartitionId();
Partition partition = olapTable.getPartition(partitionId);
if (partition == null) {
continue;
}
- long indexId = invertedIndex.getIndexId(tabletId);
+ long indexId = tabletMeta.getIndexId();
MaterializedIndex index = partition.getIndex(indexId);
if (index == null) {
continue;
@@ -512,14 +518,20 @@ public class ReportHandler extends Daemon {
try {
int deleteCounter = 0;
List<Long> tabletIds = tabletDeleteFromMeta.get(dbId);
- for (Long tabletId : tabletIds) {
- long tableId = invertedIndex.getTableId(tabletId);
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(tabletIds);
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta ==
TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ long tabletId = tabletIds.get(i);
+ long tableId = tabletMeta.getTableId();
OlapTable olapTable = (OlapTable) db.getTable(tableId);
if (olapTable == null) {
continue;
}
- long partitionId = invertedIndex.getPartitionId(tabletId);
+ long partitionId = tabletMeta.getPartitionId();
Partition partition = olapTable.getPartition(partitionId);
if (partition == null) {
continue;
@@ -527,7 +539,7 @@ public class ReportHandler extends Daemon {
short replicationNum =
olapTable.getPartitionInfo().getReplicationNum(partition.getId());
- long indexId = invertedIndex.getIndexId(tabletId);
+ long indexId = tabletMeta.getIndexId();
MaterializedIndex index = partition.getIndex(indexId);
if (index == null) {
continue;
@@ -712,10 +724,14 @@ public class ReportHandler extends Daemon {
AgentBatchTask batchTask = new AgentBatchTask();
for (TStorageMedium storageMedium : tabletMetaMigrationMap.keySet()) {
List<Long> tabletIds = tabletMetaMigrationMap.get(storageMedium);
- for (Long tabletId : tabletIds) {
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(tabletIds);
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ long tabletId = tabletIds.get(i);
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ // always get old schema hash(as effective one)
+ int effectiveSchemaHash = tabletMeta.getOldSchemaHash();
StorageMediaMigrationTask task = new
StorageMediaMigrationTask(backendId, tabletId,
-
invertedIndex.getEffectiveSchemaHash(tabletId),
- storageMedium);
+ effectiveSchemaHash, storageMedium);
batchTask.addTask(task);
}
}
@@ -763,20 +779,27 @@ public class ReportHandler extends Daemon {
}
db.writeLock();
try {
- for (Long tabletId : tabletRecoveryMap.get(dbId)) {
- long tableId = invertedIndex.getTableId(tabletId);
+ List<Long> tabletIds = tabletRecoveryMap.get(dbId);
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(tabletIds);
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta ==
TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ long tabletId = tabletIds.get(i);
+ long tableId = tabletMeta.getTableId();
OlapTable olapTable = (OlapTable) db.getTable(tableId);
if (olapTable == null) {
continue;
}
- long partitionId =
invertedIndex.getPartitionId(tabletId);
+ long partitionId = tabletMeta.getPartitionId();
Partition partition =
olapTable.getPartition(partitionId);
if (partition == null) {
continue;
}
- long indexId = invertedIndex.getIndexId(tabletId);
+ long indexId = tabletMeta.getIndexId();
MaterializedIndex index = partition.getIndex(indexId);
if (index == null) {
continue;
@@ -843,8 +866,6 @@ public class ReportHandler extends Daemon {
// need to write edit log the sync the bad info to other FEs
Catalog.getCurrentCatalog().getEditLog().logBackendTabletsInfo(backendTabletsInfo);
}
-
- return;
} else {
LOG.warn("force recovery is enable. use recovery tablet task to
recover");
AgentBatchTask batchTask = new AgentBatchTask();
@@ -892,9 +913,10 @@ public class ReportHandler extends Daemon {
}
long tabletId = tabletInfo.getTablet_id();
boolean beIsInMemory = tabletInfo.is_in_memory;
- long dbId = invertedIndex.getDbId(tabletId);
- long tableId = invertedIndex.getTableId(tabletId);
- long partitionId = invertedIndex.getPartitionId(tabletId);
+ TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId);
+ long dbId = tabletMeta != null ? tabletMeta.getDbId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
+ long tableId = tabletMeta != null ? tabletMeta.getTableId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
+ long partitionId = tabletMeta != null ?
tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE;
Database db = Catalog.getCurrentCatalog().getDb(dbId);
if (db == null) {
@@ -946,10 +968,11 @@ public class ReportHandler extends Daemon {
TabletInvertedIndex invertedIndex = Catalog.getCurrentInvertedIndex();
SystemInfoService infoService = Catalog.getCurrentSystemInfo();
- long dbId = invertedIndex.getDbId(tabletId);
- long tableId = invertedIndex.getTableId(tabletId);
- long partitionId = invertedIndex.getPartitionId(tabletId);
- long indexId = invertedIndex.getIndexId(tabletId);
+ TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId);
+ long dbId = tabletMeta != null ? tabletMeta.getDbId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
+ long tableId = tabletMeta != null ? tabletMeta.getTableId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
+ long partitionId = tabletMeta != null ? tabletMeta.getPartitionId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
+ long indexId = tabletMeta != null ? tabletMeta.getIndexId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
int schemaHash = backendTabletInfo.getSchema_hash();
long version = backendTabletInfo.getVersion();
diff --git a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 0807a85..81bf5c9 100644
--- a/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -86,6 +86,7 @@ import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
+import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.View;
import org.apache.doris.clone.DynamicPartitionScheduler;
import org.apache.doris.cluster.BaseParam;
@@ -1125,14 +1126,14 @@ public class ShowExecutor {
if (showStmt.isShowSingleTablet()) {
long tabletId = showStmt.getTabletId();
TabletInvertedIndex invertedIndex =
Catalog.getCurrentInvertedIndex();
-
- Long dbId = invertedIndex.getDbId(tabletId);
+ TabletMeta tabletMeta = invertedIndex.getTabletMeta(tabletId);
+ Long dbId = tabletMeta != null ? tabletMeta.getDbId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
String dbName = null;
- Long tableId = invertedIndex.getTableId(tabletId);
+ Long tableId = tabletMeta != null ? tabletMeta.getTableId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
String tableName = null;
- Long partitionId = invertedIndex.getPartitionId(tabletId);
+ Long partitionId = tabletMeta != null ?
tabletMeta.getPartitionId() : TabletInvertedIndex.NOT_EXIST_VALUE;
String partitionName = null;
- Long indexId = invertedIndex.getIndexId(tabletId);
+ Long indexId = tabletMeta != null ? tabletMeta.getIndexId() :
TabletInvertedIndex.NOT_EXIST_VALUE;
String indexName = null;
Boolean isSync = true;
diff --git
a/fe/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
b/fe/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
index d0b1eaa..1316ba3 100644
--- a/fe/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
+++ b/fe/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -27,6 +27,7 @@ import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
+import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DuplicatedRequestException;
@@ -367,9 +368,16 @@ public class DatabaseTransactionMgr {
// if index is dropped, it does not matter.
// if table or partition is dropped during load, just ignore that
tablet,
// because we should allow dropping rollup or partition during load
- for (TabletCommitInfo tabletCommitInfo : tabletCommitInfos) {
- long tabletId = tabletCommitInfo.getTabletId();
- long tableId = tabletInvertedIndex.getTableId(tabletId);
+ List<Long> tabletIds = tabletCommitInfos.stream().map(
+ tabletCommitInfo ->
tabletCommitInfo.getTabletId()).collect(Collectors.toList());
+ List<TabletMeta> tabletMetaList =
tabletInvertedIndex.getTabletMetaList(tabletIds);
+ for (int i = 0; i < tabletMetaList.size(); i++) {
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ long tabletId = tabletIds.get(i);
+ long tableId = tabletMeta.getTableId();
OlapTable tbl = (OlapTable) db.getTable(tableId);
if (tbl == null) {
// this can happen when tableId == -1 (tablet being dropping)
@@ -382,7 +390,7 @@ public class DatabaseTransactionMgr {
+ "Can not load into it");
}
- long partitionId = tabletInvertedIndex.getPartitionId(tabletId);
+ long partitionId = tabletMeta.getPartitionId();
if (tbl.getPartition(partitionId) == null) {
// this can happen when partitionId == -1 (tablet being
dropping)
// or partition really not exist.
@@ -396,7 +404,7 @@ public class DatabaseTransactionMgr {
if (!tabletToBackends.containsKey(tabletId)) {
tabletToBackends.put(tabletId, new HashSet<>());
}
-
tabletToBackends.get(tabletId).add(tabletCommitInfo.getBackendId());
+
tabletToBackends.get(tabletId).add(tabletCommitInfos.get(i).getBackendId());
}
if (tableToPartition.isEmpty()) {
diff --git
a/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
b/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
index 70b0e52..2371fee 100644
--- a/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
+++ b/fe/src/main/java/org/apache/doris/transaction/PublishVersionDaemon.java
@@ -20,6 +20,7 @@ package org.apache.doris.transaction;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.TabletInvertedIndex;
+import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.common.Config;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.MasterDaemon;
@@ -181,8 +182,14 @@ public class PublishVersionDaemon extends MasterDaemon {
// current we don't have partition to tablet map in
FE, so here we use an inefficient way.
// TODO(cmy): this is inefficient, but just keep it
simple. will change it later.
List<Long> tabletIds =
tabletInvertedIndex.getTabletIdsByBackendId(unfinishedTask.getBackendId());
- for (long tabletId : tabletIds) {
- long partitionId =
tabletInvertedIndex.getPartitionId(tabletId);
+ List<TabletMeta> tabletMetaList =
tabletInvertedIndex.getTabletMetaList(tabletIds);
+ for (int i = 0; i < tabletIds.size(); i++) {
+ long tabletId = tabletIds.get(i);
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta ==
TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ long partitionId = tabletMeta.getPartitionId();
if (errorPartitionIds.contains(partitionId)) {
Replica replica =
tabletInvertedIndex.getReplica(tabletId,
unfinishedTask.getBackendId());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
