This is an automated email from the ASF dual-hosted git repository.
w41ter pushed a commit to branch fix_marked_count_down
in repository https://gitbox.apache.org/repos/asf/doris.git
commit a8d1a20e95f3b6cfac71f94c82dfdf1bf8c1a653
Author: w41ter <maoch...@selectdb.com>
AuthorDate: Tue Dec 31 08:45:06 2024 +0000
[fix](catalog) Fix mark handling for failed tasks to maintain leftMarks
Many pieces of code use getLeftMarks to check whether a task is finished.
Therefore, when a task fails, its mark should not be deleted directly.
This PR addresses the issue by saving the mark of the failed task in a
separate map, maintaining the idempotency of the method while resolving
the above problem.
---
be/src/olap/data_dir.cpp | 2 +-
.../java/org/apache/doris/common/MarkedCountDownLatch.java | 10 +++++++++-
.../src/main/java/org/apache/doris/master/MasterImpl.java | 2 +-
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 4aa215e0c2e..5a773a0a6ad 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -497,7 +497,7 @@ Status DataDir::load() {
}
if (rowset_partition_id_eq_0_num >
config::ignore_invalid_partition_id_rowset_num) {
throw Exception(Status::FatalError(
- "roswet partition id eq 0 is {} bigger than config {}, be
exit, plz check be.INFO",
+ "rowset partition id eq 0 is {} bigger than config {}, be
exit, plz check be.INFO",
rowset_partition_id_eq_0_num,
config::ignore_invalid_partition_id_rowset_num));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
b/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
index e1431c4d729..5c3201e2b80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
@@ -28,11 +28,13 @@ import java.util.concurrent.CountDownLatch;
public class MarkedCountDownLatch<K, V> extends CountDownLatch {
private Multimap<K, V> marks;
+ private Multimap<K, V> failedMarks;
private Status st = Status.OK;
public MarkedCountDownLatch(int count) {
super(count);
marks = HashMultimap.create();
+ failedMarks = HashMultimap.create();
}
public synchronized void addMark(K key, V value) {
@@ -54,7 +56,13 @@ public class MarkedCountDownLatch<K, V> extends
CountDownLatch {
st = status;
}
- if (marks.remove(key, value)) {
+ // Since marks are used to determine whether a task is completed, we
should not remove
+ // a mark if the task has failed rather than finished. To maintain the
idempotency of
+ // this method, we store failed marks in a separate map.
+ //
+ // Search `getLeftMarks` for details.
+ if (!failedMarks.containsEntry(key, value)) {
+ failedMarks.put(key, value);
super.countDown();
return true;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
index b80c271bb34..ba0657e78fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
@@ -663,7 +663,7 @@ public class MasterImpl {
}
AlterInvertedIndexTask alterInvertedIndexTask =
(AlterInvertedIndexTask) task;
- LOG.info("beigin finish AlterInvertedIndexTask: {}, tablet: {},
toString: {}",
+ LOG.info("begin finish AlterInvertedIndexTask: {}, tablet: {},
toString: {}",
alterInvertedIndexTask.getSignature(),
alterInvertedIndexTask.getTabletId(),
alterInvertedIndexTask.toString());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
