This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new b167c8006f1 branch-3.1: [fix](mtmv) release read lock when align
mvmv's partition #53069 (#53509)
b167c8006f1 is described below
commit b167c8006f1513e8908f93f5f3fa1150df204604
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Jul 19 09:25:27 2025 +0800
branch-3.1: [fix](mtmv) release read lock when align mvmv's partition
#53069 (#53509)
Cherry-picked from #53069
Co-authored-by: Lijia Liu <[email protected]>
Co-authored-by: liutang123 <[email protected]>
---
.../org/apache/doris/job/extensions/mtmv/MTMVTask.java | 18 +++++++++++++++++-
.../java/org/apache/doris/mtmv/MTMVPartitionUtil.java | 16 ++++++++++------
2 files changed, 27 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
index ca865d8d43b..4df48187cb3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java
@@ -17,6 +17,7 @@
package org.apache.doris.job.extensions.mtmv;
+import org.apache.doris.analysis.PartitionKeyDesc;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Env;
import org.apache.doris.catalog.MTMV;
@@ -27,6 +28,7 @@ import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.Pair;
import org.apache.doris.common.Status;
import org.apache.doris.common.UserException;
import org.apache.doris.common.util.DebugPointUtil;
@@ -196,6 +198,7 @@ public class MTMVTask extends AbstractTask {
tableIfs.sort(Comparator.comparing(TableIf::getId));
MTMVRefreshContext context;
+ Pair<List<String>, List<PartitionKeyDesc>> syncPartitions = null;
// lock table order by id to avoid deadlock
MetaLockUtils.readLockTables(tableIfs);
try {
@@ -212,8 +215,21 @@ public class MTMVTask extends AbstractTask {
+ " e.g. Table has multiple partition columns"
+ " or including not supported transform
functions.");
}
- MTMVPartitionUtil.alignMvPartition(mtmv);
+ syncPartitions = MTMVPartitionUtil.alignMvPartition(mtmv);
}
+ } finally {
+ MetaLockUtils.readUnlockTables(tableIfs);
+ }
+ if (syncPartitions != null) {
+ for (String pName : syncPartitions.first) {
+ MTMVPartitionUtil.dropPartition(mtmv, pName);
+ }
+ for (PartitionKeyDesc partitionKeyDesc :
syncPartitions.second) {
+ MTMVPartitionUtil.addPartition(mtmv, partitionKeyDesc);
+ }
+ }
+ MetaLockUtils.readLockTables(tableIfs);
+ try {
context = MTMVRefreshContext.buildContext(mtmv);
this.needRefreshPartitions =
calculateNeedRefreshPartitions(context);
} finally {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
index 8484b07ae0d..82305aad7d7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java
@@ -33,6 +33,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.Pair;
import org.apache.doris.datasource.mvcc.MvccUtil;
import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType;
import org.apache.doris.rpc.RpcException;
@@ -47,6 +48,7 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -114,24 +116,26 @@ public class MTMVPartitionUtil {
* @throws DdlException
* @throws AnalysisException
*/
- public static void alignMvPartition(MTMV mtmv)
- throws DdlException, AnalysisException {
+ public static Pair<List<String>, List<PartitionKeyDesc>>
alignMvPartition(MTMV mtmv) throws AnalysisException {
Map<String, PartitionKeyDesc> mtmvPartitionDescs =
mtmv.generateMvPartitionDescs();
Set<PartitionKeyDesc> relatedPartitionDescs =
generateRelatedPartitionDescs(mtmv.getMvPartitionInfo(),
mtmv.getMvProperties()).keySet();
+ List<String> partitionsToDrop = new ArrayList<>();
+ List<PartitionKeyDesc> partitionsToAdd = new ArrayList<>();
// drop partition of mtmv
for (Entry<String, PartitionKeyDesc> entry :
mtmvPartitionDescs.entrySet()) {
if (!relatedPartitionDescs.contains(entry.getValue())) {
- dropPartition(mtmv, entry.getKey());
+ partitionsToDrop.add(entry.getKey());
}
}
// add partition for mtmv
HashSet<PartitionKeyDesc> mtmvPartitionDescsSet =
Sets.newHashSet(mtmvPartitionDescs.values());
for (PartitionKeyDesc desc : relatedPartitionDescs) {
if (!mtmvPartitionDescsSet.contains(desc)) {
- addPartition(mtmv, desc);
+ partitionsToAdd.add(desc);
}
}
+ return Pair.of(partitionsToDrop, partitionsToAdd);
}
/**
@@ -366,7 +370,7 @@ public class MTMVPartitionUtil {
* @param mtmv
* @param partitionName
*/
- private static void dropPartition(MTMV mtmv, String partitionName) throws
DdlException {
+ public static void dropPartition(MTMV mtmv, String partitionName) throws
DdlException {
if (!mtmv.writeLockIfExist()) {
return;
}
@@ -387,7 +391,7 @@ public class MTMVPartitionUtil {
* @param oldPartitionKeyDesc
* @throws DdlException
*/
- private static void addPartition(MTMV mtmv, PartitionKeyDesc
oldPartitionKeyDesc)
+ public static void addPartition(MTMV mtmv, PartitionKeyDesc
oldPartitionKeyDesc)
throws DdlException {
Map<String, String> partitionProperties = Maps.newHashMap();
SinglePartitionDesc singlePartitionDesc = new SinglePartitionDesc(true,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
