zhannngchen commented on code in PR #49383:
URL: https://github.com/apache/doris/pull/49383#discussion_r2081053906
##########
cloud/src/recycler/checker.cpp:
##########
@@ -1165,14 +1172,119 @@ int
InstanceChecker::check_delete_bitmap_storage_optimize(int64_t tablet_id) {
return (abnormal_rowsets_num > 1 ? 1 : 0);
}
-int InstanceChecker::do_delete_bitmap_storage_optimize_check() {
+int InstanceChecker::check_delete_bitmap_storage_optimize_v2(int64_t
tablet_id) {
+ // number of rowsets which may have problems
+ int64_t abnormal_rowsets_num {0};
+
+ // [end_version, create_time]
+ std::map<int64_t, int64_t> tablet_rowsets_map {};
+ // Get all visible rowsets of this tablet
+ auto collect_cb = [&tablet_rowsets_map](const doris::RowsetMetaCloudPB&
rowset) {
+ if (rowset.start_version() == 0 && rowset.end_version() == 1) {
+ // ignore dummy rowset [0-1]
+ return;
+ }
+ tablet_rowsets_map[rowset.end_version()] = rowset.creation_time();
+ };
+ if (int ret = collect_tablet_rowsets(tablet_id, collect_cb); ret != 0) {
+ return ret;
+ }
+
+ std::unique_ptr<RangeGetIterator> it;
+ auto begin = meta_delete_bitmap_key({instance_id_, tablet_id, "", 0, 0});
+ auto end = meta_delete_bitmap_key({instance_id_, tablet_id + 1, "", 0, 0});
+ std::string last_rowset_id = "";
+ int64_t last_version = 0;
+ std::string last_failed_rowset_id = "";
+ using namespace std::chrono;
+ int64_t now =
duration_cast<seconds>(system_clock::now().time_since_epoch()).count();
+ do {
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to create txn";
+ return -1;
+ }
+ err = txn->get(begin, end, &it);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to get delete bitmap kv, err=" << err;
+ return -1;
+ }
+ if (!it->has_next()) {
+ break;
+ }
+ while (it->has_next() && !stopped()) {
+ auto [k, v] = it->next();
+ std::string_view k1 = k;
+ k1.remove_prefix(1);
+ std::vector<std::tuple<std::variant<int64_t, std::string>, int,
int>> out;
+ decode_key(&k1, &out);
+ // 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id}
${rowset_id} ${version} ${segment_id} -> roaringbitmap
+ auto rowset_id = std::get<std::string>(std::get<0>(out[4]));
+ auto version = std::get<std::int64_t>(std::get<0>(out[5]));
+ if (!it->has_next()) {
+ begin = k;
+ begin.push_back('\x00'); // Update to next smallest key for
iteration
+ }
+ if (rowset_id == last_rowset_id && version == last_version) {
+ // skip the same rowset and version
+ continue;
+ }
+ if (tablet_rowsets_map.find(version) == tablet_rowsets_map.end()) {
+ // there may be an interval in this situation:
+ // 1. finish compaction job; 2. checker; 3. finish agg and
remove delete bitmap to ms
+ auto rowset_it = tablet_rowsets_map.upper_bound(version);
+ if (rowset_it != tablet_rowsets_map.end()) {
+ if (rowset_it->second +
+
config::delete_bitmap_storage_optimize_v2_check_skip_seconds >=
+ now) {
+ LOG(INFO) << fmt::format(
Review Comment:
should be DEBUG level?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
