This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 6dbf3494f88 [fix](merge-cloud) Fix drop rollup/mv leak meta kvs and
data objects resource (#33321)
6dbf3494f88 is described below
commit 6dbf3494f88df30bafc0b46ef9590acc9481207f
Author: Lei Zhang <27994433+swjtu-zhang...@users.noreply.github.com>
AuthorDate: Mon Apr 8 08:44:32 2024 +0800
[fix](merge-cloud) Fix drop rollup/mv leak meta kvs and data objects
resource (#33321)
---
.../doris/alter/MaterializedViewHandler.java | 11 ++++++++
.../cloud/datasource/CloudInternalCatalog.java | 31 ++++++++++++++++++++++
.../apache/doris/datasource/InternalCatalog.java | 4 +++
3 files changed, 46 insertions(+)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
index 82388d62348..e09b1de4a72 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/alter/MaterializedViewHandler.java
@@ -86,6 +86,7 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
/*
* MaterializedViewHandler is responsible for ADD/DROP materialized view.
@@ -912,6 +913,7 @@ public class MaterializedViewHandler extends AlterHandler {
public void processBatchDropRollup(List<AlterClause> dropRollupClauses,
Database db, OlapTable olapTable)
throws DdlException, MetaNotFoundException {
+ List<Long> deleteIndexList = null;
olapTable.writeLockOrDdlException();
try {
olapTable.checkNormalStateForAlter();
@@ -942,15 +944,18 @@ public class MaterializedViewHandler extends AlterHandler
{
long tableId = olapTable.getId();
String tableName = olapTable.getName();
editLog.logBatchDropRollup(new BatchDropInfo(dbId, tableId,
tableName, indexIdSet));
+ deleteIndexList = indexIdSet.stream().collect(Collectors.toList());
LOG.info("finished drop rollup index[{}] in table[{}]",
String.join("", rollupNameSet), olapTable.getName());
} finally {
olapTable.writeUnlock();
}
+ Env.getCurrentInternalCatalog().eraseDroppedIndex(olapTable.getId(),
deleteIndexList);
}
public void processDropMaterializedView(DropMaterializedViewStmt
dropMaterializedViewStmt, Database db,
OlapTable olapTable) throws DdlException, MetaNotFoundException {
+ List<Long> deleteIndexList = new ArrayList<Long>();
olapTable.writeLockOrDdlException();
try {
olapTable.checkNormalStateForAlter();
@@ -963,6 +968,7 @@ public class MaterializedViewHandler extends AlterHandler {
EditLog editLog = Env.getCurrentEnv().getEditLog();
editLog.logDropRollup(
new DropInfo(db.getId(), olapTable.getId(),
olapTable.getName(), mvIndexId, false, 0));
+ deleteIndexList.add(mvIndexId);
LOG.info("finished drop materialized view [{}] in table [{}]",
mvName, olapTable.getName());
} catch (MetaNotFoundException e) {
if (dropMaterializedViewStmt.isIfExists()) {
@@ -973,6 +979,7 @@ public class MaterializedViewHandler extends AlterHandler {
} finally {
olapTable.writeUnlock();
}
+ Env.getCurrentInternalCatalog().eraseDroppedIndex(olapTable.getId(),
deleteIndexList);
}
/**
@@ -1060,6 +1067,10 @@ public class MaterializedViewHandler extends
AlterHandler {
} finally {
olapTable.writeUnlock();
}
+
+ List<Long> deleteIndexList = new ArrayList<Long>();
+ deleteIndexList.add(rollupIndexId);
+ Env.getCurrentInternalCatalog().eraseDroppedIndex(olapTable.getId(),
deleteIndexList);
LOG.info("replay drop rollup {}", dropInfo.getIndexId());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
index 3b7d27f1596..5c0509831ad 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/cloud/datasource/CloudInternalCatalog.java
@@ -701,6 +701,37 @@ public class CloudInternalCatalog extends InternalCatalog {
}
}
+ /**
+ * for cloud mode, drop rollup/materializedIndex in kv meta store
+ * @param tableId
+ * @param indexIdList
+ */
+ public void eraseDroppedIndex(long tableId, List<Long> indexIdList) {
+ if (indexIdList == null || indexIdList.size() == 0) {
+ LOG.warn("indexIdList is empty");
+ return;
+ }
+ long tryCnt = 0;
+ while (true) {
+ if (tryCnt++ > Config.drop_rpc_retry_num) {
+ LOG.warn("failed to drop index {} of table {}, try cnt {}
reaches maximum retry count",
+ indexIdList, tableId, tryCnt);
+ break;
+ }
+
+ try {
+ dropMaterializedIndex(tableId, indexIdList, false);
+ break;
+ } catch (Exception e) {
+ LOG.warn("tryCnt:{}, eraseDroppedIndex exception:", tryCnt, e);
+ }
+ sleepSeveralMs();
+ }
+
+ LOG.info("eraseDroppedIndex finished, tableId:{}, indexIdList:{}",
+ tableId, indexIdList);
+ }
+
// END DROP TABLE
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
index fb5ea1a869f..d19704753ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalCatalog.java
@@ -1023,6 +1023,10 @@ public class InternalCatalog implements
CatalogIf<Database> {
// no need send be delete task, when be report its tablets, fe will
send delete task then.
}
+ public void eraseDroppedIndex(long tableId, List<Long> indexIdList) {
+ // nothing to do in non cloud mode
+ }
+
private void unprotectAddReplica(OlapTable olapTable, ReplicaPersistInfo
info) {
if (LOG.isDebugEnabled()) {
LOG.debug("replay add a replica {}", info);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
