github-actions[bot] commented on code in PR #44317:
URL: https://github.com/apache/doris/pull/44317#discussion_r1849594298
##########
cloud/src/recycler/checker.h:
##########
@@ -23,13 +23,18 @@
#include <atomic>
#include <condition_variable>
#include <deque>
+#include <functional>
#include <thread>
#include <unordered_map>
#include <unordered_set>
#include "recycler/storage_vault_accessor.h"
Review Comment:
warning: 'recycler/storage_vault_accessor.h' file not found
[clang-diagnostic-error]
```cpp
#include "recycler/storage_vault_accessor.h"
^
```
##########
cloud/test/recycler_test.cpp:
##########
@@ -2576,6 +2626,352 @@
}
}
+TEST(CheckerTest, delete_bitmap_inverted_check_normal) {
+ // normal case, all delete bitmaps belong to a rowset
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+ constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+ // create some rowsets with delete bitmaps in merge-on-write tablet
+ for (int tablet_id = 600001; tablet_id <= 600010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ int64_t rowset_start_id = 400;
+ for (int ver = 2; ver <= 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ if (ver >= 5) {
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val {"test"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+ } else {
+ // delete bitmaps may be spilitted into mulitiple KVs if too
large
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val(1000, 'A');
+ cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0,
300);
+ }
+ }
+ }
+
+ // also create some rowsets without delete bitmaps in non merge-on-write
tablet
+ for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 500;
+ for (int ver = 2; ver < 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ }
+ }
+
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+ ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 0);
+}
+
+TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) {
+ // abnormal case, some delete bitmaps arem leaked
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ // tablet_id -> [rowset_id, version, segment_id]
+ std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+ expected_abnormal_delete_bitmaps {}, real_abnormal_delete_bitmaps
{};
+ std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+ expected_leaked_delete_bitmaps {}, real_leaked_delete_bitmaps {};
+ auto sp = SyncPoint::get_instance();
Review Comment:
warning: 'auto sp' can be declared as 'auto *sp' [readability-qualified-auto]
```suggestion
auto *sp = SyncPoint::get_instance();
```
##########
cloud/test/recycler_test.cpp:
##########
@@ -2576,6 +2626,352 @@
}
}
+TEST(CheckerTest, delete_bitmap_inverted_check_normal) {
+ // normal case, all delete bitmaps belong to a rowset
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+ constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+ // create some rowsets with delete bitmaps in merge-on-write tablet
+ for (int tablet_id = 600001; tablet_id <= 600010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ int64_t rowset_start_id = 400;
+ for (int ver = 2; ver <= 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ if (ver >= 5) {
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val {"test"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+ } else {
+ // delete bitmaps may be spilitted into mulitiple KVs if too
large
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val(1000, 'A');
+ cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0,
300);
+ }
+ }
+ }
+
+ // also create some rowsets without delete bitmaps in non merge-on-write
tablet
+ for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 500;
+ for (int ver = 2; ver < 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ }
+ }
+
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+ ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 0);
+}
+
+TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) {
+ // abnormal case, some delete bitmaps arem leaked
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ // tablet_id -> [rowset_id, version, segment_id]
+ std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+ expected_abnormal_delete_bitmaps {}, real_abnormal_delete_bitmaps
{};
+ std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+ expected_leaked_delete_bitmaps {}, real_leaked_delete_bitmaps {};
+ auto sp = SyncPoint::get_instance();
+ std::unique_ptr<int, std::function<void(int*)>> defer(
+ (int*)0x01, [](int*) {
SyncPoint::get_instance()->clear_all_call_backs(); });
+ sp->set_call_back(
+
"InstanceChecker::do_delete_bitmap_inverted_check.get_abnormal_delete_bitmap",
+ [&real_abnormal_delete_bitmaps](auto&& args) {
+ int64_t tablet_id = *try_any_cast<int64_t*>(args[0]);
+ std::string rowset_id = *try_any_cast<std::string*>(args[1]);
+ int64_t version = *try_any_cast<int64_t*>(args[2]);
+ int64_t segment_id = *try_any_cast<int64_t*>(args[3]);
+ real_abnormal_delete_bitmaps[tablet_id].insert({rowset_id,
version, segment_id});
+ });
+ sp->set_call_back(
+
"InstanceChecker::do_delete_bitmap_inverted_check.get_leaked_delete_bitmap",
+ [&real_leaked_delete_bitmaps](auto&& args) {
+ int64_t tablet_id = *try_any_cast<int64_t*>(args[0]);
+ std::string rowset_id = *try_any_cast<std::string*>(args[1]);
+ int64_t version = *try_any_cast<int64_t*>(args[2]);
+ int64_t segment_id = *try_any_cast<int64_t*>(args[3]);
+ real_leaked_delete_bitmaps[tablet_id].insert({rowset_id,
version, segment_id});
+ });
+ sp->enable_processing();
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+ constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+ // create some rowsets with delete bitmaps in merge-on-write tablet
+ for (int tablet_id = 800001; tablet_id <= 800010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ int64_t rowset_start_id = 600;
+ for (int ver = 2; ver <= 20; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+
+ if (ver >= 10) {
+ // only create rowsets for some versions
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id,
false, 1);
+ } else {
+ expected_leaked_delete_bitmaps[tablet_id].insert({rowset_id,
ver, 0});
+ }
+
+ if (ver >= 5) {
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val {"test"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+ } else {
+ // delete bitmaps may be spilitted into mulitiple KVs if too
large
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val(1000, 'A');
+ cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0,
300);
+ }
+ }
+ }
+
+ // create some rowsets with delete bitmaps in non merge-on-write tablet
+ for (int tablet_id = 900001; tablet_id <= 900010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 700;
+ for (int ver = 2; ver < 6; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id, rowset_id,
ver, 0});
+ std::string delete_bitmap_val {"test2"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+
+ expected_abnormal_delete_bitmaps[tablet_id].insert({rowset_id,
ver, 0});
+ }
+ }
+
+ // create some rowsets without delete bitmaps in non merge-on-write tablet
+ for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 500;
+ for (int ver = 2; ver < 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ }
+ }
+
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+ ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 1);
+ ASSERT_EQ(expected_leaked_delete_bitmaps, real_leaked_delete_bitmaps);
+ ASSERT_EQ(expected_abnormal_delete_bitmaps, real_abnormal_delete_bitmaps);
+}
+
+TEST(CheckerTest, delete_bitmap_storage_optimize_check_normal) {
+ config::delete_bitmap_storage_optimize_check_version_gap = 0;
+
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+ constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+ int64_t rowset_start_id = 600;
+
+ for (int tablet_id = 800001; tablet_id <= 800005; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ std::vector<std::pair<int64_t, int64_t>> rowset_vers {{2, 2}, {3, 3},
{4, 4}, {5, 5},
+ {6, 7}, {8, 8},
{9, 9}};
+ std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+ {7, 9}, {8, 9}, {7, 9}, {7, 9}, {7, 9}, {8, 9}, {9, 9}};
+ std::vector<bool> segments_overlap {true, true, true, true, false,
true, true};
+ for (size_t i {0}; i < 7; i++) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ rowset_vers[i].first,
rowset_vers[i].second,
+ rowset_id,
segments_overlap[i], 1);
+ create_delete_bitmaps(txn.get(), tablet_id, rowset_id,
delete_bitmaps_vers[i].first,
+ delete_bitmaps_vers[i].second);
+ }
+ }
+
+ for (int tablet_id = 800006; tablet_id <= 800010; ++tablet_id) {
+ // [7-7] cumu compaction output rowset start_version == end_version
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ std::vector<std::pair<int64_t, int64_t>> rowset_vers {{2, 2}, {3, 3},
{4, 4}, {5, 5},
+ {6, 6}, {7, 7},
{8, 8}, {9, 9}};
+ std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+ {7, 9}, {8, 9}, {7, 9}, {7, 9}, {7, 9}, {7, 9}, {8, 9}, {9,
9}};
+ std::vector<bool> segments_overlap {true, true, false, true, false,
true, true, true};
+ for (size_t i {0}; i < 8; i++) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ rowset_vers[i].first,
rowset_vers[i].second,
+ rowset_id,
segments_overlap[i], 1);
+ create_delete_bitmaps(txn.get(), tablet_id, rowset_id,
delete_bitmaps_vers[i].first,
+ delete_bitmaps_vers[i].second);
+ }
+ }
+
+ for (int tablet_id = 800011; tablet_id <= 800015; ++tablet_id) {
+ // no rowsets are compacted
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ std::vector<std::pair<int64_t, int64_t>> rowset_vers {{2, 2}, {3, 3},
{4, 4}, {5, 5},
+ {6, 6}, {7, 7},
{8, 8}, {9, 9}};
+ std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+ {2, 9}, {3, 9}, {4, 9}, {5, 9}, {6, 9}, {7, 9}, {8, 9}, {9,
9}};
+ std::vector<bool> segments_overlap {true, true, true, true, true,
true, true, true};
+ for (size_t i {0}; i < 8; i++) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ rowset_vers[i].first,
rowset_vers[i].second,
+ rowset_id,
segments_overlap[i], 1);
+ create_delete_bitmaps(txn.get(), tablet_id, rowset_id,
delete_bitmaps_vers[i].first,
+ delete_bitmaps_vers[i].second);
+ }
+ }
+
+ for (int tablet_id = 800016; tablet_id <= 800020; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ std::vector<std::pair<int64_t, int64_t>> rowset_vers {
+ {2, 5}, {6, 6}, {7, 7}, {8, 8}, {9, 9}};
+ std::vector<std::pair<int64_t, int64_t>> delete_bitmaps_vers {
+ {5, 9}, {6, 9}, {7, 9}, {8, 9}, {9, 9}};
+ std::vector<bool> segments_overlap {false, true, true, true, true};
+ for (size_t i {0}; i < 5; i++) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ rowset_vers[i].first,
rowset_vers[i].second,
+ rowset_id,
segments_overlap[i], 1);
+ create_delete_bitmaps(txn.get(), tablet_id, rowset_id,
delete_bitmaps_vers[i].first,
+ delete_bitmaps_vers[i].second);
+ }
+ }
+
+ // also create some rowsets without delete bitmaps in non merge-on-write
tablet
+ for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 500;
+ for (int ver = 2; ver < 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ }
+ }
+
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+ ASSERT_EQ(checker.do_delete_bitmap_storage_optimize_check(), 0);
+}
+
+TEST(CheckerTest, delete_bitmap_storage_optimize_check_abnormal) {
+ config::delete_bitmap_storage_optimize_check_version_gap = 0;
+ // abnormal case, some rowsets' delete bitmaps are not deleted as expected
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ // tablet_id -> [rowset_id]
+ std::map<std::int64_t, std::set<std::string>> expected_abnormal_rowsets {};
+ std::map<std::int64_t, std::set<std::string>> real_abnormal_rowsets {};
+ auto sp = SyncPoint::get_instance();
Review Comment:
warning: 'auto sp' can be declared as 'auto *sp' [readability-qualified-auto]
```suggestion
auto *sp = SyncPoint::get_instance();
```
##########
cloud/test/recycler_test.cpp:
##########
@@ -2576,6 +2626,352 @@ TEST(CheckerTest, do_inspect) {
}
}
+TEST(CheckerTest, delete_bitmap_inverted_check_normal) {
+ // normal case, all delete bitmaps belong to a rowset
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+ constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+ // create some rowsets with delete bitmaps in merge-on-write tablet
+ for (int tablet_id = 600001; tablet_id <= 600010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ int64_t rowset_start_id = 400;
+ for (int ver = 2; ver <= 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ if (ver >= 5) {
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val {"test"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+ } else {
+ // delete bitmaps may be spilitted into mulitiple KVs if too
large
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val(1000, 'A');
+ cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0,
300);
+ }
+ }
+ }
+
+ // also create some rowsets without delete bitmaps in non merge-on-write
tablet
+ for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 500;
+ for (int ver = 2; ver < 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ }
+ }
+
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+ ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 0);
+}
+
+TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) {
Review Comment:
warning: function 'TEST' exceeds recommended size/complexity thresholds
[readability-function-size]
```cpp
TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) {
^
```
<details>
<summary>Additional context</summary>
**cloud/test/recycler_test.cpp:2687:** 112 lines including whitespace and
comments (threshold 80)
```cpp
TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) {
^
```
</details>
##########
cloud/test/recycler_test.cpp:
##########
@@ -2576,6 +2626,352 @@
}
}
+TEST(CheckerTest, delete_bitmap_inverted_check_normal) {
+ // normal case, all delete bitmaps belong to a rowset
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+ constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+ // create some rowsets with delete bitmaps in merge-on-write tablet
+ for (int tablet_id = 600001; tablet_id <= 600010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ int64_t rowset_start_id = 400;
+ for (int ver = 2; ver <= 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ if (ver >= 5) {
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val {"test"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+ } else {
+ // delete bitmaps may be spilitted into mulitiple KVs if too
large
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val(1000, 'A');
+ cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0,
300);
+ }
+ }
+ }
+
+ // also create some rowsets without delete bitmaps in non merge-on-write
tablet
+ for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 500;
+ for (int ver = 2; ver < 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ }
+ }
+
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+ ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 0);
+}
+
+TEST(CheckerTest, delete_bitmap_inverted_check_abnormal) {
+ // abnormal case, some delete bitmaps arem leaked
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+ auto accessor = checker.accessor_map_.begin()->second;
+
+ // tablet_id -> [rowset_id, version, segment_id]
+ std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+ expected_abnormal_delete_bitmaps {}, real_abnormal_delete_bitmaps
{};
+ std::map<std::int64_t, std::set<std::tuple<std::string, int64_t, int64_t>>>
+ expected_leaked_delete_bitmaps {}, real_leaked_delete_bitmaps {};
+ auto sp = SyncPoint::get_instance();
+ std::unique_ptr<int, std::function<void(int*)>> defer(
+ (int*)0x01, [](int*) {
SyncPoint::get_instance()->clear_all_call_backs(); });
+ sp->set_call_back(
+
"InstanceChecker::do_delete_bitmap_inverted_check.get_abnormal_delete_bitmap",
+ [&real_abnormal_delete_bitmaps](auto&& args) {
+ int64_t tablet_id = *try_any_cast<int64_t*>(args[0]);
+ std::string rowset_id = *try_any_cast<std::string*>(args[1]);
+ int64_t version = *try_any_cast<int64_t*>(args[2]);
+ int64_t segment_id = *try_any_cast<int64_t*>(args[3]);
+ real_abnormal_delete_bitmaps[tablet_id].insert({rowset_id,
version, segment_id});
+ });
+ sp->set_call_back(
+
"InstanceChecker::do_delete_bitmap_inverted_check.get_leaked_delete_bitmap",
+ [&real_leaked_delete_bitmaps](auto&& args) {
+ int64_t tablet_id = *try_any_cast<int64_t*>(args[0]);
+ std::string rowset_id = *try_any_cast<std::string*>(args[1]);
+ int64_t version = *try_any_cast<int64_t*>(args[2]);
+ int64_t segment_id = *try_any_cast<int64_t*>(args[3]);
+ real_leaked_delete_bitmaps[tablet_id].insert({rowset_id,
version, segment_id});
+ });
+ sp->enable_processing();
+
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn_kv->create_txn(&txn));
+
+ constexpr int table_id = 10000, index_id = 10001, partition_id = 10002;
+ // create some rowsets with delete bitmaps in merge-on-write tablet
+ for (int tablet_id = 800001; tablet_id <= 800010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, true));
+ int64_t rowset_start_id = 600;
+ for (int ver = 2; ver <= 20; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+
+ if (ver >= 10) {
+ // only create rowsets for some versions
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id,
false, 1);
+ } else {
+ expected_leaked_delete_bitmaps[tablet_id].insert({rowset_id,
ver, 0});
+ }
+
+ if (ver >= 5) {
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val {"test"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+ } else {
+ // delete bitmaps may be spilitted into mulitiple KVs if too
large
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id,
rowset_id, ver, 0});
+ std::string delete_bitmap_val(1000, 'A');
+ cloud::put(txn.get(), delete_bitmap_key, delete_bitmap_val, 0,
300);
+ }
+ }
+ }
+
+ // create some rowsets with delete bitmaps in non merge-on-write tablet
+ for (int tablet_id = 900001; tablet_id <= 900010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 700;
+ for (int ver = 2; ver < 6; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ auto delete_bitmap_key =
+ meta_delete_bitmap_key({instance_id, tablet_id, rowset_id,
ver, 0});
+ std::string delete_bitmap_val {"test2"};
+ txn->put(delete_bitmap_key, delete_bitmap_val);
+
+ expected_abnormal_delete_bitmaps[tablet_id].insert({rowset_id,
ver, 0});
+ }
+ }
+
+ // create some rowsets without delete bitmaps in non merge-on-write tablet
+ for (int tablet_id = 700001; tablet_id <= 700010; ++tablet_id) {
+ ASSERT_EQ(0,
+ create_tablet(txn_kv.get(), table_id, index_id,
partition_id, tablet_id, false));
+ int64_t rowset_start_id = 500;
+ for (int ver = 2; ver < 10; ++ver) {
+ std::string rowset_id = std::to_string(rowset_start_id++);
+ create_committed_rowset_with_rowset_id(txn_kv.get(),
accessor.get(), "1", tablet_id,
+ ver, ver, rowset_id, false,
1);
+ }
+ }
+
+ ASSERT_EQ(TxnErrorCode::TXN_OK, txn->commit());
+
+ ASSERT_EQ(checker.do_delete_bitmap_inverted_check(), 1);
+ ASSERT_EQ(expected_leaked_delete_bitmaps, real_leaked_delete_bitmaps);
+ ASSERT_EQ(expected_abnormal_delete_bitmaps, real_abnormal_delete_bitmaps);
+}
+
+TEST(CheckerTest, delete_bitmap_storage_optimize_check_normal) {
Review Comment:
warning: function 'TEST' exceeds recommended size/complexity thresholds
[readability-function-size]
```cpp
TEST(CheckerTest, delete_bitmap_storage_optimize_check_normal) {
^
```
<details>
<summary>Additional context</summary>
**cloud/test/recycler_test.cpp:2801:** 109 lines including whitespace and
comments (threshold 80)
```cpp
TEST(CheckerTest, delete_bitmap_storage_optimize_check_normal) {
^
```
</details>
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
