gavinchou commented on code in PR #40204:
URL: https://github.com/apache/doris/pull/40204#discussion_r1766327188
##########
cloud/src/meta-service/meta_service.cpp:
##########
@@ -2080,6 +2080,146 @@ void
MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
}
}
+void
MetaServiceImpl::update_delete_bitmap_without_lock(google::protobuf::RpcController*
controller,
+ const
UpdateDeleteBitmapRequest* request,
+
UpdateDeleteBitmapResponse* response,
+
::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(update_delete_bitmap_without_lock);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ return;
+ }
+ RPC_RATE_LIMIT(update_delete_bitmap_without_lock)
+
+ uint32_t fdb_txn_size = 0;
+ auto tablet_id = request->tablet_id();
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+
+ PendingDeleteBitmapPB delete_bitmap_keys;
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ MetaDeleteBitmapInfo key_info {instance_id, tablet_id,
request->rowset_ids(i),
+ request->versions(i),
request->segment_ids(i)};
+ std::string key;
+ meta_delete_bitmap_key(key_info, &key);
+ delete_bitmap_keys.add_delete_bitmap_keys(key);
+ }
+
+ // remove old delete bitmap
+
+ // Update delete bitmap for curent txn
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ auto& key = delete_bitmap_keys.delete_bitmap_keys(i);
+ auto& val = request->segment_delete_bitmaps(i);
+
+ // Split into multiple fdb transactions, because the size of one fdb
+ // transaction can't exceed 10MB.
+ if (fdb_txn_size + key.size() + val.size() > 9 * 1024 * 1024) {
+ LOG(INFO) << "fdb txn size more than 9MB, current size: " <<
fdb_txn_size
+ << " lock_id=" << request->lock_id();
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err;
+ msg = ss.str();
+ return;
+ }
+ fdb_txn_size = 0;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+ }
+ // splitting large values (>90*1000) into multiple KVs
+ cloud::put(txn.get(), key, val, 0);
+ fdb_txn_size = fdb_txn_size + key.size() + val.size();
+ LOG(INFO) << "xxx update delete bitmap put delete_bitmap_key=" <<
hex(key)
Review Comment:
there may be too many keys.
print the first or the last one, an sumarized how many keys and size have
been put.
##########
cloud/src/meta-service/meta_service.cpp:
##########
@@ -2080,6 +2080,146 @@ void
MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
}
}
+void
MetaServiceImpl::update_delete_bitmap_without_lock(google::protobuf::RpcController*
controller,
+ const
UpdateDeleteBitmapRequest* request,
+
UpdateDeleteBitmapResponse* response,
+
::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(update_delete_bitmap_without_lock);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ return;
+ }
+ RPC_RATE_LIMIT(update_delete_bitmap_without_lock)
+
+ uint32_t fdb_txn_size = 0;
+ auto tablet_id = request->tablet_id();
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+
+ PendingDeleteBitmapPB delete_bitmap_keys;
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ MetaDeleteBitmapInfo key_info {instance_id, tablet_id,
request->rowset_ids(i),
+ request->versions(i),
request->segment_ids(i)};
+ std::string key;
+ meta_delete_bitmap_key(key_info, &key);
+ delete_bitmap_keys.add_delete_bitmap_keys(key);
+ }
+
+ // remove old delete bitmap
+
+ // Update delete bitmap for curent txn
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ auto& key = delete_bitmap_keys.delete_bitmap_keys(i);
+ auto& val = request->segment_delete_bitmaps(i);
+
+ // Split into multiple fdb transactions, because the size of one fdb
+ // transaction can't exceed 10MB.
+ if (fdb_txn_size + key.size() + val.size() > 9 * 1024 * 1024) {
+ LOG(INFO) << "fdb txn size more than 9MB, current size: " <<
fdb_txn_size
+ << " lock_id=" << request->lock_id();
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err;
+ msg = ss.str();
+ return;
+ }
+ fdb_txn_size = 0;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+ }
+ // splitting large values (>90*1000) into multiple KVs
+ cloud::put(txn.get(), key, val, 0);
+ fdb_txn_size = fdb_txn_size + key.size() + val.size();
+ LOG(INFO) << "xxx update delete bitmap put delete_bitmap_key=" <<
hex(key)
Review Comment:
there may be too many keys.
print the first or the last one, an summarize how many keys and size have
been put.
##########
cloud/src/meta-service/meta_service.cpp:
##########
@@ -2080,6 +2080,146 @@ void
MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
}
}
+void
MetaServiceImpl::update_delete_bitmap_without_lock(google::protobuf::RpcController*
controller,
+ const
UpdateDeleteBitmapRequest* request,
+
UpdateDeleteBitmapResponse* response,
+
::google::protobuf::Closure* done) {
+ RPC_PREPROCESS(update_delete_bitmap_without_lock);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
+ return;
+ }
+ RPC_RATE_LIMIT(update_delete_bitmap_without_lock)
+
+ uint32_t fdb_txn_size = 0;
+ auto tablet_id = request->tablet_id();
+
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+
+ PendingDeleteBitmapPB delete_bitmap_keys;
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ MetaDeleteBitmapInfo key_info {instance_id, tablet_id,
request->rowset_ids(i),
+ request->versions(i),
request->segment_ids(i)};
+ std::string key;
+ meta_delete_bitmap_key(key_info, &key);
+ delete_bitmap_keys.add_delete_bitmap_keys(key);
+ }
+
+ // remove old delete bitmap
+
+ // Update delete bitmap for curent txn
+ for (size_t i = 0; i < request->rowset_ids_size(); ++i) {
+ auto& key = delete_bitmap_keys.delete_bitmap_keys(i);
+ auto& val = request->segment_delete_bitmaps(i);
+
+ // Split into multiple fdb transactions, because the size of one fdb
+ // transaction can't exceed 10MB.
+ if (fdb_txn_size + key.size() + val.size() > 9 * 1024 * 1024) {
+ LOG(INFO) << "fdb txn size more than 9MB, current size: " <<
fdb_txn_size
+ << " lock_id=" << request->lock_id();
+ err = txn->commit();
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::COMMIT>(err);
+ ss << "failed to update delete bitmap, err=" << err;
+ msg = ss.str();
+ return;
+ }
+ fdb_txn_size = 0;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ code = cast_as<ErrCategory::CREATE>(err);
+ msg = "failed to init txn";
+ return;
+ }
+ }
+ // splitting large values (>90*1000) into multiple KVs
+ cloud::put(txn.get(), key, val, 0);
+ fdb_txn_size = fdb_txn_size + key.size() + val.size();
+ LOG(INFO) << "xxx update delete bitmap put delete_bitmap_key=" <<
hex(key)
Review Comment:
there may be too many keys.
print the first or the last one, and summarize how many keys and size have
been put.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
