This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 0b4b7311a09 branch-3.1: [fix](mow) delete bitmap is not deleted if
commit compaction job failed #56758 (#56785)
0b4b7311a09 is described below
commit 0b4b7311a0925fc6b3a840c4d549f550d1fc453d
Author: meiyi <[email protected]>
AuthorDate: Mon Oct 13 12:05:06 2025 +0800
branch-3.1: [fix](mow) delete bitmap is not deleted if commit compaction
job failed #56758 (#56785)
picked from #56758
---
be/src/cloud/cloud_meta_mgr.cpp | 3 +++
cloud/src/recycler/recycler.cpp | 20 ++++++++++++++++++++
cloud/test/recycler_test.cpp | 23 ++++++++++++++++++++++-
3 files changed, 45 insertions(+), 1 deletion(-)
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index fb68c6dfc80..1d73d4ed745 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -1373,6 +1373,9 @@ Status CloudMetaMgr::prepare_tablet_job(const
TabletJobInfoPB& job, StartTabletJ
Status CloudMetaMgr::commit_tablet_job(const TabletJobInfoPB& job,
FinishTabletJobResponse* res) {
VLOG_DEBUG << "commit_tablet_job: " << job.ShortDebugString();
TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::commit_tablet_job",
Status::OK(), job, res);
+ DBUG_EXECUTE_IF("CloudMetaMgr::commit_tablet_job.fail", {
+ return Status::InternalError<false>("inject
CloudMetaMgr::commit_tablet_job.fail");
+ });
FinishTabletJobRequest req;
req.mutable_job()->CopyFrom(job);
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index c084aa3956b..a3a306a0a83 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -2892,6 +2892,19 @@ int InstanceRecycler::recycle_tmp_rowsets() {
return 0;
};
+ auto delete_delete_bitmap_kvs = [&](int64_t tablet_id, const std::string&
rowset_id) {
+ auto delete_bitmap_start =
+ meta_delete_bitmap_key({instance_id_, tablet_id, rowset_id, 0,
0});
+ auto delete_bitmap_end =
+ meta_delete_bitmap_key({instance_id_, tablet_id, rowset_id,
INT64_MAX, INT64_MAX});
+ auto ret = txn_remove(txn_kv_.get(), delete_bitmap_start,
delete_bitmap_end);
+ if (ret != 0) {
+ LOG(WARNING) << "failed to delete delete bitmap kv, instance_id="
<< instance_id_
+ << ", tablet_id=" << tablet_id << ", rowset_id=" <<
rowset_id;
+ }
+ return ret;
+ };
+
auto loop_done = [&]() -> int {
DORIS_CLOUD_DEFER {
tmp_rowset_keys.clear();
@@ -2904,6 +2917,13 @@ int InstanceRecycler::recycle_tmp_rowsets() {
LOG(WARNING) << "failed to delete tmp rowset data,
instance_id=" << instance_id_;
return;
}
+ for (const auto& [_, rs] : tmp_rowsets_to_delete) {
+ if (delete_delete_bitmap_kvs(rs.tablet_id(),
rs.rowset_id_v2()) != 0) {
+ LOG(WARNING) << "failed to delete delete bitmap kv, rs="
+ << rs.ShortDebugString();
+ return;
+ }
+ }
if (txn_remove(txn_kv_.get(), tmp_rowset_keys_to_delete) != 0) {
LOG(WARNING) << "failed to tmp rowset kv, instance_id=" <<
instance_id_;
return;
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index 634c78c061e..b9ff08f7a17 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -1104,6 +1104,16 @@ static int get_copy_file_num(TxnKv* txn_kv, const
std::string& stage_id, int64_t
return 0;
}
+static void check_delete_bitmap_keys_size(TxnKv* txn_kv, int64_t tablet_id,
int expected_size) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::unique_ptr<RangeGetIterator> it;
+ auto dbm_start_key = meta_delete_bitmap_key({instance_id, tablet_id, "",
0, 0});
+ auto dbm_end_key = meta_delete_bitmap_key({instance_id, tablet_id + 1, "",
0, 0});
+ ASSERT_EQ(txn->get(dbm_start_key, dbm_end_key, &it), TxnErrorCode::TXN_OK);
+ EXPECT_EQ(it->size(), expected_size);
+}
+
TEST(RecyclerTest, recycle_empty) {
auto txn_kv = std::make_shared<MemTxnKv>();
ASSERT_EQ(txn_kv->init(), 0);
@@ -1328,14 +1338,23 @@ TEST(RecyclerTest, recycle_tmp_rowsets) {
int64_t txn_id_base = 114115;
int64_t tablet_id_base = 10015;
int64_t index_id_base = 1000;
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
for (int i = 0; i < 50; ++i) {
int64_t txn_id = txn_id_base + i;
for (int j = 0; j < 1000; ++j) {
auto rowset = create_rowset("recycle_tmp_rowsets", tablet_id_base
+ j,
index_id_base + j, 5, schemas[i % 5],
txn_id);
create_tmp_rowset(txn_kv.get(), accessor.get(), rowset, i & 1,
false);
+ if (i < 50) {
+ create_delete_bitmaps(txn.get(), tablet_id_base + j,
rowset.rowset_id_v2(), 0, 1);
+ }
}
}
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ for (int j = 0; j < 20; ++j) {
+ check_delete_bitmap_keys_size(txn_kv.get(), tablet_id_base + j, 100);
+ }
auto start = std::chrono::steady_clock::now();
ASSERT_EQ(recycler.recycle_tmp_rowsets(), 0);
@@ -1349,7 +1368,6 @@ TEST(RecyclerTest, recycle_tmp_rowsets) {
ASSERT_EQ(0, accessor->list_directory("data/", &list_iter));
ASSERT_FALSE(list_iter->has_next());
// check all tmp rowset kv have been deleted
- std::unique_ptr<Transaction> txn;
ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
std::unique_ptr<RangeGetIterator> it;
auto begin_key = meta_rowset_tmp_key({instance_id, 0, 0});
@@ -1360,6 +1378,9 @@ TEST(RecyclerTest, recycle_tmp_rowsets) {
// EXPECT_GE for InvertedIndexIdCache::get
EXPECT_GE(insert_inverted_index, 4000);
EXPECT_GE(insert_no_inverted_index, 1000);
+ for (int j = 0; j < 20; ++j) {
+ check_delete_bitmap_keys_size(txn_kv.get(), tablet_id_base + j, 0);
+ }
}
TEST(RecyclerTest, recycle_tmp_rowsets_partial_update) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
