yujun777 commented on code in PR #42401:
URL: https://github.com/apache/doris/pull/42401#discussion_r1823663812
##########
fe/fe-core/src/main/java/org/apache/doris/alter/SystemHandler.java:
##########
@@ -193,19 +209,115 @@ public synchronized void process(String rawSql,
List<AlterClause> alterClauses,
/*
* check if the specified backends can be dropped
* 1. backend does not have any tablet.
- * 2. all tablets in backend have been recycled.
+ * 2. all tablets in backend have been recycled or leaky.
+ *
+ * sampleTablets: sample normal tablets
+ * sampleLeakyTablets: sample leaky tablets
+ *
*/
- private boolean checkTablets(Long beId, List<Long> backendTabletIds) {
+ private boolean checkMigrateTablets(long beId, int sampleLimit, List<Long>
sampleTablets,
+ List<Long> sampleLeakyTablets, AtomicInteger totalTabletNum) {
+ TabletInvertedIndex invertedIndex = Env.getCurrentInvertedIndex();
+ List<Long> backendTabletIds =
invertedIndex.getTabletIdsByBackendId(beId);
+ totalTabletNum.set(backendTabletIds.size());
if (backendTabletIds.isEmpty()) {
return true;
}
- if (backendTabletIds.size() <
Config.decommission_tablet_check_threshold
- &&
Env.getCurrentRecycleBin().allTabletsInRecycledStatus(backendTabletIds)) {
- LOG.info("tablet size is {}, all tablets on decommissioned backend
{} have been recycled,"
- + " so this backend will be dropped immediately",
backendTabletIds.size(), beId);
- return true;
+ // if too many tablets, no check for efficiency
+ if (backendTabletIds.size() >
Config.decommission_tablet_check_threshold) {
+ return false;
+ }
+ // dbId -> tableId -> partitionId -> tablet list
+ Map<Long, Map<Long, Map<Long, List<Long>>>> tabletsMap =
Maps.newHashMap();
+ List<TabletMeta> tabletMetaList =
invertedIndex.getTabletMetaList(backendTabletIds);
+ for (int i = 0; i < backendTabletIds.size(); i++) {
+ long tabletId = backendTabletIds.get(i);
+ TabletMeta tabletMeta = tabletMetaList.get(i);
+ if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
+ continue;
+ }
+ tabletsMap.computeIfAbsent(tabletMeta.getDbId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getTableId(), k ->
Maps.newHashMap())
+ .computeIfAbsent(tabletMeta.getPartitionId(), k ->
Lists.newArrayList())
+ .add(tabletId);
+ }
+ InternalCatalog catalog = Env.getCurrentInternalCatalog();
+ CatalogRecycleBin recycleBin = Env.getCurrentRecycleBin();
+ long now = System.currentTimeMillis();
+ Map<Long, Long> leakyTablets = Maps.newHashMap();
+ boolean searchedFirstTime = !backendLeakyTablets.containsKey(beId);
+ Map<Long, Long> lastLeakyTablets =
backendLeakyTablets.computeIfAbsent(beId, k -> Maps.newHashMap());
+ backendLeakyTablets.put(beId, leakyTablets);
+ Consumer<List<Long>> addPartitionLeakyTablets = tabletsOfPartition -> {
+ tabletsOfPartition.forEach(tabletId -> {
+ leakyTablets.put(tabletId,
lastLeakyTablets.getOrDefault(tabletId, now));
+ });
+ };
+ Consumer<Map<Long, List<Long>>> addTableLeakyTablets = tabletsOfTable
-> {
+ tabletsOfTable.values().forEach(addPartitionLeakyTablets);
+ };
+ Consumer<Map<Long, Map<Long, List<Long>>>> addDbLeakyTablets =
tabletsOfDb -> {
+ tabletsOfDb.values().forEach(addTableLeakyTablets);
+ };
+ boolean searchedAllTablets = true;
+ OUTER:
+ for (Map.Entry<Long, Map<Long, Map<Long, List<Long>>>> dbEntry :
tabletsMap.entrySet()) {
+ long dbId = dbEntry.getKey();
+ Database db = catalog.getDbNullable(dbId);
+ if (db == null) {
+ if (!recycleBin.isRecycleDatabase(dbId)) {
+ addDbLeakyTablets.accept(dbEntry.getValue());
+ }
+ continue;
+ }
+
+ for (Map.Entry<Long, Map<Long, List<Long>>> tableEntry :
dbEntry.getValue().entrySet()) {
+ long tableId = tableEntry.getKey();
+ Table tbl = db.getTableNullable(tableId);
+ if (tbl == null || !tbl.isManagedTable()) {
+ if (!recycleBin.isRecycleTable(dbId, tableId)) {
+ addTableLeakyTablets.accept(tableEntry.getValue());
+ }
+ continue;
+ }
+
+ OlapTable olapTable = (OlapTable) tbl;
Review Comment:
这里应该还好，只有第一次检查时是搜索全量的tablet。 后面每次检查最多只搜索10个正常的tablet，搜索即停止。
这里的泄漏tablet 应该不多的， 再加上只搜索10个tablet即提前终止搜索，时间应该很小
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
