This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cb6287e2acb [fix](path-gc) Fix pending rowset guard check failure when
ordered data compaction failed (#33026)
cb6287e2acb is described below
commit cb6287e2acbe881a9bb240db5e3a7d5ffff4ded4
Author: plat1ko <platonekos...@gmail.com>
AuthorDate: Fri Mar 29 19:45:28 2024 +0800
[fix](path-gc) Fix pending rowset guard check failure when ordered data
compaction failed (#33026)
---
be/src/olap/compaction.cpp | 11 ++++++++++
be/src/olap/compaction.h | 1 +
be/src/olap/rowset/pending_rowset_helper.cpp | 31 ++++++++++++++++++++++++++++
be/src/olap/rowset/pending_rowset_helper.h | 22 +++++++-------------
4 files changed, 50 insertions(+), 15 deletions(-)
diff --git a/be/src/olap/compaction.cpp b/be/src/olap/compaction.cpp
index e43e5e05c29..6efd308a8b3 100644
--- a/be/src/olap/compaction.cpp
+++ b/be/src/olap/compaction.cpp
@@ -319,6 +319,12 @@ bool CompactionMixin::handle_ordered_data_compaction() {
_tablet->enable_unique_key_merge_on_write()) {
return false;
}
+
+ if (_tablet->tablet_meta()->tablet_schema()->skip_write_index_on_load()) {
+ // Expected to create index through normal compaction
+ return false;
+ }
+
// check delete version: if compaction type is base compaction and
// has a delete version, use original compaction
if (compaction_type() == ReaderType::READER_BASE_COMPACTION) {
@@ -346,6 +352,11 @@ bool CompactionMixin::handle_ordered_data_compaction() {
// most rowset of current compaction is nonoverlapping
// just handle nonoverlappint rowsets
auto st = do_compact_ordered_rowsets();
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to compact ordered rowsets: " << st;
+ _pending_rs_guard.drop();
+ }
+
return st.ok();
}
diff --git a/be/src/olap/compaction.h b/be/src/olap/compaction.h
index 75837e83b0e..69c64adcb23 100644
--- a/be/src/olap/compaction.h
+++ b/be/src/olap/compaction.h
@@ -147,6 +147,7 @@ private:
void construct_skip_inverted_index(RowsetWriterContext& ctx);
+ // Return true if do ordered data compaction successfully
bool handle_ordered_data_compaction();
Status do_compact_ordered_rowsets();
diff --git a/be/src/olap/rowset/pending_rowset_helper.cpp
b/be/src/olap/rowset/pending_rowset_helper.cpp
index 3e976344f6f..6c788b84232 100644
--- a/be/src/olap/rowset/pending_rowset_helper.cpp
+++ b/be/src/olap/rowset/pending_rowset_helper.cpp
@@ -17,6 +17,8 @@
#include "olap/rowset/pending_rowset_helper.h"
+#include "olap/olap_common.h"
+
namespace doris {
PendingRowsetGuard::~PendingRowsetGuard() {
@@ -28,6 +30,35 @@ PendingRowsetGuard::~PendingRowsetGuard() {
PendingRowsetGuard::PendingRowsetGuard(const RowsetId& rowset_id,
PendingRowsetSet* set)
: _rowset_id(rowset_id), _pending_rowset_set(set) {}
+PendingRowsetGuard::PendingRowsetGuard(PendingRowsetGuard&& other) noexcept {
+ CHECK(!_pending_rowset_set ||
+ (_rowset_id == other._rowset_id && _pending_rowset_set ==
other._pending_rowset_set))
+ << _rowset_id << ' ' << other._rowset_id << ' ' <<
_pending_rowset_set << ' '
+ << other._pending_rowset_set;
+ _rowset_id = other._rowset_id;
+ _pending_rowset_set = other._pending_rowset_set;
+ other._pending_rowset_set = nullptr;
+}
+
+PendingRowsetGuard& PendingRowsetGuard::operator=(PendingRowsetGuard&& other)
noexcept {
+ CHECK(!_pending_rowset_set ||
+ (_rowset_id == other._rowset_id && _pending_rowset_set ==
other._pending_rowset_set))
+ << _rowset_id << ' ' << other._rowset_id << ' ' <<
_pending_rowset_set << ' '
+ << other._pending_rowset_set;
+ _rowset_id = other._rowset_id;
+ _pending_rowset_set = other._pending_rowset_set;
+ other._pending_rowset_set = nullptr;
+ return *this;
+}
+
+void PendingRowsetGuard::drop() {
+ if (_pending_rowset_set) {
+ _pending_rowset_set->remove(_rowset_id);
+ }
+ _pending_rowset_set = nullptr;
+ _rowset_id = RowsetId {};
+}
+
bool PendingRowsetSet::contains(const RowsetId& rowset_id) {
std::lock_guard lock(_mtx);
return _set.contains(rowset_id);
diff --git a/be/src/olap/rowset/pending_rowset_helper.h
b/be/src/olap/rowset/pending_rowset_helper.h
index 53d1f4f16c5..360424636b9 100644
--- a/be/src/olap/rowset/pending_rowset_helper.h
+++ b/be/src/olap/rowset/pending_rowset_helper.h
@@ -35,21 +35,13 @@ public:
PendingRowsetGuard(const PendingRowsetGuard&) = delete;
PendingRowsetGuard& operator=(const PendingRowsetGuard&) = delete;
- PendingRowsetGuard(PendingRowsetGuard&& other) noexcept {
- CHECK(!_pending_rowset_set ||
- (_rowset_id == other._rowset_id && _pending_rowset_set ==
other._pending_rowset_set));
- _rowset_id = other._rowset_id;
- _pending_rowset_set = other._pending_rowset_set;
- other._pending_rowset_set = nullptr;
- }
- PendingRowsetGuard& operator=(PendingRowsetGuard&& other) noexcept {
- CHECK(!_pending_rowset_set ||
- (_rowset_id == other._rowset_id && _pending_rowset_set ==
other._pending_rowset_set));
- _rowset_id = other._rowset_id;
- _pending_rowset_set = other._pending_rowset_set;
- other._pending_rowset_set = nullptr;
- return *this;
- }
+ PendingRowsetGuard(PendingRowsetGuard&& other) noexcept;
+ PendingRowsetGuard& operator=(PendingRowsetGuard&& other) noexcept;
+
+ // Remove guarded rowset id from `PendingRowsetSet` if it's initialized
and reset the guard to
+ // uninitialized state. This be used to manually release the pending
rowset, ensure that the
+ // guarded rowset is indeed no longer in use.
+ void drop();
private:
friend class PendingRowsetSet;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
