This is an automated email from the ASF dual-hosted git repository.
gavinchou pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 22eb777099a [fix](catalog) Fix mark handling for failed tasks to
maintain getLeftMarks (#46205)
22eb777099a is described below
commit 22eb777099a99aefef47d68c9eddd13d298cc96c
Author: walter <maoch...@selectdb.com>
AuthorDate: Tue Dec 31 19:59:56 2024 +0800
[fix](catalog) Fix mark handling for failed tasks to maintain getLeftMarks
(#46205)
Many pieces of code use `getLeftMarks` to check whether a task is
finished. Therefore, when a task fails, its mark should not be deleted
directly. For example, when creating a tablet, the left marks are used
to track how many replicas were successfully created. If the number of
successful replicas exceeds the quorum, the tablet is considered
successfully created. However, if the mark is deleted due to failure, it
could lead to a situation where the majority of replicas fail to be
created, but the tablet is still mistakenly considered as successfully
created.
This PR addresses the issue by saving the mark of the failed task in a
separate map while maintaining the method's idempotency.
---
be/src/olap/data_dir.cpp | 2 +-
.../java/org/apache/doris/common/MarkedCountDownLatch.java | 10 +++++++++-
.../src/main/java/org/apache/doris/master/MasterImpl.java | 2 +-
3 files changed, 11 insertions(+), 3 deletions(-)
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 4aa215e0c2e..5a773a0a6ad 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -497,7 +497,7 @@ Status DataDir::load() {
}
if (rowset_partition_id_eq_0_num >
config::ignore_invalid_partition_id_rowset_num) {
throw Exception(Status::FatalError(
- "roswet partition id eq 0 is {} bigger than config {}, be
exit, plz check be.INFO",
+ "rowset partition id eq 0 is {} bigger than config {}, be
exit, plz check be.INFO",
rowset_partition_id_eq_0_num,
config::ignore_invalid_partition_id_rowset_num));
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
b/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
index e1431c4d729..5c3201e2b80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/MarkedCountDownLatch.java
@@ -28,11 +28,13 @@ import java.util.concurrent.CountDownLatch;
public class MarkedCountDownLatch<K, V> extends CountDownLatch {
private Multimap<K, V> marks;
+ private Multimap<K, V> failedMarks;
private Status st = Status.OK;
public MarkedCountDownLatch(int count) {
super(count);
marks = HashMultimap.create();
+ failedMarks = HashMultimap.create();
}
public synchronized void addMark(K key, V value) {
@@ -54,7 +56,13 @@ public class MarkedCountDownLatch<K, V> extends
CountDownLatch {
st = status;
}
- if (marks.remove(key, value)) {
+ // Since marks are used to determine whether a task is completed, we
should not remove
+ // a mark if the task has failed rather than finished. To maintain the
idempotency of
+ // this method, we store failed marks in a separate map.
+ //
+ // Search `getLeftMarks` for details.
+ if (!failedMarks.containsEntry(key, value)) {
+ failedMarks.put(key, value);
super.countDown();
return true;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
index b80c271bb34..ba0657e78fb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/master/MasterImpl.java
@@ -663,7 +663,7 @@ public class MasterImpl {
}
AlterInvertedIndexTask alterInvertedIndexTask =
(AlterInvertedIndexTask) task;
- LOG.info("beigin finish AlterInvertedIndexTask: {}, tablet: {},
toString: {}",
+ LOG.info("begin finish AlterInvertedIndexTask: {}, tablet: {},
toString: {}",
alterInvertedIndexTask.getSignature(),
alterInvertedIndexTask.getTabletId(),
alterInvertedIndexTask.toString());
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
