gavinchou commented on code in PR #40204:
URL: https://github.com/apache/doris/pull/40204#discussion_r1766345496
##########
gensrc/proto/cloud.proto:
##########
@@ -1550,6 +1562,8 @@ service MetaService {
rpc update_delete_bitmap(UpdateDeleteBitmapRequest)
returns(UpdateDeleteBitmapResponse);
rpc get_delete_bitmap(GetDeleteBitmapRequest)
returns(GetDeleteBitmapResponse);
rpc get_delete_bitmap_update_lock(GetDeleteBitmapUpdateLockRequest)
returns(GetDeleteBitmapUpdateLockResponse);
+ rpc update_delete_bitmap_without_lock(UpdateDeleteBitmapRequest)
returns(UpdateDeleteBitmapResponse);
Review Comment:
why we cannot use the existing rpc `rpc
update_delete_bitmap(UpdateDeleteBitmapRequest)`?
##########
cloud/src/meta-service/meta_service.cpp:
##########
@@ -2080,6 +2080,146 @@ void
MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
}
}
+void
MetaServiceImpl::update_delete_bitmap_without_lock(google::protobuf::RpcController*
controller,
+ const
UpdateDeleteBitmapRequest* request,
+
UpdateDeleteBitmapResponse* response,
+
::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(update_delete_bitmap_without_lock);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ return;
+ }
+ RPC_RATE_LIMIT(update_delete_bitmap_without_lock)
+
+ uint32_t fdb_txn_size = 0;
+ auto tablet_id = request->tablet_id();
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+
+ PendingDeleteBitmapPB delete_bitmap_keys;
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ MetaDeleteBitmapInfo key_info {instance_id, tablet_id,
request->rowset_ids(i),
+ request->versions(i),
request->segment_ids(i)};
+ std::string key;
+ meta_delete_bitmap_key(key_info, &key);
+ delete_bitmap_keys.add_delete_bitmap_keys(key);
+ }
+
+ // remove old delete bitmap
+
+ // Update delete bitmap for curent txn
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ auto& key = delete_bitmap_keys.delete_bitmap_keys(i);
+ auto& val = request->segment_delete_bitmaps(i);
+
+ // Split into multiple fdb transactions, because the size of one fdb
+ // transaction can't exceed 10MB.
+ if (fdb_txn_size + key.size() + val.size() > 9 * 1024 * 1024) {
+ LOG(INFO) << "fdb txn size more than 9MB, current size: " <<
fdb_txn_size
+ << " lock_id=" << request->lock_id();
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err;
+ msg = ss.str();
+ return;
+ }
+ fdb_txn_size = 0;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+ }
+ // splitting large values (>90*1000) into multiple KVs
+ cloud::put(txn.get(), key, val, 0);
+ fdb_txn_size = fdb_txn_size + key.size() + val.size();
+ LOG(INFO) << "xxx update delete bitmap put delete_bitmap_key=" <<
hex(key)
+ << " lock_id=" << request->lock_id() << " value_size: " <<
val.size();
+ }
+
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err;
+ msg = ss.str();
+ return;
+ }
+}
+
+void MetaServiceImpl::remove_delete_bitmap(google::protobuf::RpcController*
controller,
+ const RemoveDeleteBitmapRequest*
request,
+ RemoveDeleteBitmapResponse*
response,
+ ::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(remove_delete_bitmap);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ return;
+ }
+ RPC_RATE_LIMIT(remove_delete_bitmap)
+ // remove delete bitmap of input rowset for MoW table
+ auto tablet_id = request->tablet_id();
+ auto& rowset_ids = request->rowset_ids();
+ auto& begin_versions = request->begin_versions();
+ auto& end_versions = request->end_versions();
+ if (rowset_ids.size() != begin_versions.size() || rowset_ids.size() !=
end_versions.size()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ ss << "rowset and version size not match. "
+ << " rowset_size=" << rowset_ids.size()
+ << " begin_version_size=" << begin_versions.size()
+ << " end_version_size=" << end_versions.size();
+ msg = ss.str();
+ return;
+ }
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to init txn";
+ return;
+ }
+ for (size_t i = 0; i < rowset_ids.size(); i++) {
+ auto delete_bitmap_start = meta_delete_bitmap_key(
Review Comment:
can we estimate how many keys in the range, if there are few of them.
try to remove by keys.
And, we should summarize and log what have been removed, say, how many
keys/ranges have been removed.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
