hust-hhb commented on code in PR #40204:
URL: https://github.com/apache/doris/pull/40204#discussion_r1766764941
##########
cloud/src/meta-service/meta_service.cpp:
##########
@@ -2080,6 +2080,146 @@ void
MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
}
}
+void
MetaServiceImpl::update_delete_bitmap_without_lock(google::protobuf::RpcController*
controller,
+ const
UpdateDeleteBitmapRequest* request,
+
UpdateDeleteBitmapResponse* response,
+
::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(update_delete_bitmap_without_lock);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ return;
+ }
+ RPC_RATE_LIMIT(update_delete_bitmap_without_lock)
+
+ uint32_t fdb_txn_size = 0;
+ auto tablet_id = request->tablet_id();
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+
+ PendingDeleteBitmapPB delete_bitmap_keys;
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ MetaDeleteBitmapInfo key_info {instance_id, tablet_id,
request->rowset_ids(i),
+ request->versions(i),
request->segment_ids(i)};
+ std::string key;
+ meta_delete_bitmap_key(key_info, &key);
+ delete_bitmap_keys.add_delete_bitmap_keys(key);
+ }
+
+ // remove old delete bitmap
+
+ // Update delete bitmap for curent txn
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ auto& key = delete_bitmap_keys.delete_bitmap_keys(i);
+ auto& val = request->segment_delete_bitmaps(i);
+
+ // Split into multiple fdb transactions, because the size of one fdb
+ // transaction can't exceed 10MB.
+ if (fdb_txn_size + key.size() + val.size() > 9 * 1024 * 1024) {
+ LOG(INFO) << "fdb txn size more than 9MB, current size: " <<
fdb_txn_size
+ << " lock_id=" << request->lock_id();
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err;
+ msg = ss.str();
+ return;
+ }
+ fdb_txn_size = 0;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+ }
+ // splitting large values (>90*1000) into multiple KVs
+ cloud::put(txn.get(), key, val, 0);
+ fdb_txn_size = fdb_txn_size + key.size() + val.size();
+ LOG(INFO) << "xxx update delete bitmap put delete_bitmap_key=" <<
hex(key)
+ << " lock_id=" << request->lock_id() << " value_size: " <<
val.size();
+ }
+
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err;
+ msg = ss.str();
+ return;
+ }
+}
+
+void MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController*
controller,
+ const RemoveDeleteBitmapRequest*
request,
+ RemoveDeleteBitmapResponse*
response,
+ ::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(remove_delete_bitmap);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ return;
+ }
+ RPC_RATE_LIMIT(remove_delete_bitmap)
+ // remove delete bitmap of input rowset for MoW table
+ auto tablet_id = request->tablet_id();
+ auto& rowset_ids = request->rowset_ids();
+ auto& begin_versions = request->begin_versions();
+ auto& end_versions = request->end_versions();
+ if (rowset_ids.size() != begin_versions.size() || rowset_ids.size() !=
end_versions.size()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ ss << "rowset and version size not match. "
+ << " rowset_size=" << rowset_ids.size()
+ << " begin_version_size=" << begin_versions.size()
+ << " end_version_size=" << end_versions.size();
+ msg = ss.str();
+ return;
+ }
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to init txn";
+ return;
+ }
+ for (size_t i = 0; i < rowset_ids.size(); i++) {
+ auto delete_bitmap_start = meta_delete_bitmap_key(
Review Comment:
now can not estimate how many keys beacause the request param is provide
segment nums, may be in next time, we can remove delete bitmap key in more
accurate range.
in meta_service_job.cpp, remove delete bitmap code is
// remove delete bitmap of input rowset for MoW table
if (compaction.has_delete_bitmap_lock_initiator()) {
auto delete_bitmap_start =
meta_delete_bitmap_key({instance_id, tablet_id,
rs.rowset_id_v2(), 0, 0});
auto delete_bitmap_end = meta_delete_bitmap_key(
{instance_id, tablet_id, rs.rowset_id_v2(),
INT64_MAX, INT64_MAX});
txn->remove(delete_bitmap_start, delete_bitmap_end);
}
and in get_delete_bitmap function, get delete bitmap code is
MetaDeleteBitmapInfo start_key_info {instance_id, tablet_id,
rowset_ids[i],
begin_versions[i], 0};
MetaDeleteBitmapInfo end_key_info {instance_id, tablet_id,
rowset_ids[i], end_versions[i],
INT64_MAX};
std::string start_key;
std::string end_key;
meta_delete_bitmap_key(start_key_info, &start_key);
meta_delete_bitmap_key(end_key_info, &end_key);
those key range is more bigger, we can make these key range more accurate in
next time.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
