deardeng commented on code in PR #42401:
URL: https://github.com/apache/doris/pull/42401#discussion_r1822548779
##########
fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java:
##########
@@ -193,19 +209,115 @@ public synchronized void process(String rawSql,
List<AlterClause> alterClauses,
/*
* check if the specified backends can be dropped
* 1. backend does not have any tablet.
- * 2. all tablets in backend have been recycled.
+ * 2. all tablets in backend have been recycled or leaky.
+ *
+ * sampleTablets: sample normal tablets
+ * sampleLeakyTablets: sample leaky tablets
+ *
*/
- private boolean checkTablets(Long beId, List<Long> backendTabletIds) {
+ private boolean checkMigrateTablets(long beId, int sampleLimit, List<Long>
sampleTablets,
+ List<Long> sampleLeakyTablets, AtomicInteger totalTabletNum) {
+ TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
+ List<Long> backendTabletIds =
invertedIndex.getTabletIdsByBackendId(beId);
+ totalTabletNum.set(backendTabletIds.size());
if (backendTabletIds.isEmpty()) {
return true;
}
- if (backendTabletIds.size() <
Config.decommission_tablet_check_threshold
- &&
Env.getCurrentRecycleBin().allTabletsInRecycledStatus(backendTabletIds)) {
- LOG.info("tablet size is {}, all tablets on decommissioned backend
{} have been recycled,"
- + " so this backend will be dropped immediately",
backendTabletIds.size(), beId);
- return true;
+ // if too many tablets, no check for efficiency
+ if (backendTabletIds.size() >
Config.decommission_tablet_check_threshold) {
+ return false;
+ }
+ // dbId -> tableId -> partitionId -> tablet list
+ Map<Long, Map<Long, Map<Long, List<Long>>>> tabletsMap =
Maps.newHashMap();
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(backendTabletIds);
+ for (int i = 0; i < backendTabletIds.size(); i++) {
+ long tabletId = backendTabletIds.get(i);
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ tabletsMap.computeIfAbsent(tabletMeta.getDbId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getTableId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getPartitionId(), k ->
Lists.newArrayList())
+ .add(tabletId);
+ }
+ InternalCatalog catalog = Env.getCurrentInternalCatalog();
+ CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin();
+ long now = System.currentTimeMillis();
+ Map<Long, Long> leakyTablets = Maps.newHashMap();
+ boolean searchedFirstTime = !backendLeakyTablets.containsKey(beId);
+ Map<Long, Long> lastLeakyTablets =
backendLeakyTablets.computeIfAbsent(beId, k -> Maps.newHashMap());
+ backendLeakyTablets.put(beId, leakyTablets);
+ Consumer<List<Long>> addPartitionLeakyTablets = tabletsOfPartition -> {
+ tabletsOfPartition.forEach(tabletId -> {
+ leakyTablets.put(tabletId,
lastLeakyTablets.getOrDefault(tabletId, now));
+ });
+ };
+ Consumer<Map<Long, List<Long>>> addTableLeakyTablets = tabletsOfTable
-> {
+ tabletsOfTable.values().forEach(addPartitionLeakyTablets);
+ };
+ Consumer<Map<Long, Map<Long, List<Long>>>> addDbLeakyTablets =
tabletsOfDb -> {
+ tabletsOfDb.values().forEach(addTableLeakyTablets);
+ };
+ boolean searchedAllTablets = true;
+ OUTER:
+ for (Map.Entry<Long, Map<Long, Map<Long, List<Long>>>> dbEntry :
tabletsMap.entrySet()) {
+ long dbId = dbEntry.getKey();
+ Database db = catalog.getDbNullable(dbId);
+ if (db == null) {
+ if (!recycleBin.isRecycleDatabase(dbId)) {
+ addDbLeakyTablets.accept(dbEntry.getValue());
+ }
+ continue;
+ }
+
+ for (Map.Entry<Long, Map<Long, List<Long>>> tableEntry :
dbEntry.getValue().entrySet()) {
+ long tableId = tableEntry.getKey();
+ Table tbl = db.getTableNullable(tableId);
+ if (tbl == null || !tbl.isManagedTable()) {
+ if (!recycleBin.isRecycleTable(dbId, tableId)) {
+ addTableLeakyTablets.accept(tableEntry.getValue());
+ }
+ continue;
+ }
+
+ OlapTable olapTable = (OlapTable) tbl;
Review Comment:
表锁粒度是不有些大了？partitions很多的情况下，表锁一直lock？
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
