This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new d016936db75 branch-3.1: [improve](cloud-mow)Add white list for delete
bitmap update lock version #50173 (#52147)
d016936db75 is described below
commit d016936db75386829cd1c67f733599af4051c3aa
Author: meiyi <[email protected]>
AuthorDate: Tue Jun 24 10:51:24 2025 +0800
branch-3.1: [improve](cloud-mow)Add white list for delete bitmap update
lock version #50173 (#52147)
Cherry-pick #50173
Co-authored-by: huanghaibin <[email protected]>
---
cloud/src/common/config.h | 7 +-
cloud/src/common/configbase.cpp | 9 +-
cloud/src/common/configbase.h | 3 +
cloud/src/meta-service/CMakeLists.txt | 1 +
.../meta-service/delete_bitmap_lock_white_list.cpp | 84 +++
.../meta-service/delete_bitmap_lock_white_list.h | 47 ++
cloud/src/meta-service/meta_service.cpp | 153 ++--
cloud/src/meta-service/meta_service.h | 24 +-
cloud/src/meta-service/meta_service_job.cpp | 36 +-
cloud/test/meta_service_http_test.cpp | 114 +++
cloud/test/meta_service_job_test.cpp | 825 ++++++++++++++++++---
cloud/test/recycler_test.cpp | 2 +-
12 files changed, 1088 insertions(+), 217 deletions(-)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index df5856365d1..bbf40cd3826 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -288,7 +288,12 @@ CONF_Bool(enable_check_instance_id, "true");
CONF_Bool(enable_loopback_address_for_ms, "false");
// delete_bitmap_lock version config
-CONF_mString(use_delete_bitmap_lock_version, "v1");
+// here is some examples:
+// 1. If instance1,instance2 use v2, config should be
+// delete_bitmap_lock_v2_white_list = instance1;instance2
+// 2. If all instance use v2, config should be
+// delete_bitmap_lock_v2_white_list = *
+CONF_mString(delete_bitmap_lock_v2_white_list, "");
// FOR DEBUGGING
CONF_mBool(use_delete_bitmap_lock_random_version, "false");
diff --git a/cloud/src/common/configbase.cpp b/cloud/src/common/configbase.cpp
index cacbf3479d8..ae32ffb5f4d 100644
--- a/cloud/src/common/configbase.cpp
+++ b/cloud/src/common/configbase.cpp
@@ -26,6 +26,7 @@
#include <list>
#include <map>
#include <mutex>
+#include <shared_mutex>
#include <sstream>
#include <utility>
@@ -38,7 +39,7 @@ namespace doris::cloud::config {
std::map<std::string, Register::Field>* Register::_s_field_map = nullptr;
std::map<std::string, std::function<bool()>>*
RegisterConfValidator::_s_field_validator = nullptr;
std::map<std::string, std::string>* full_conf_map = nullptr;
-std::mutex mutable_string_config_lock;
+std::shared_mutex mutable_string_config_lock;
std::mutex conf_persist_lock;
// trim string
@@ -406,7 +407,7 @@ bool do_set_config(const Register::Field& feild, const
std::string& value, bool
UPDATE_FIELD(feild, value, double, need_persist);
{
// add lock to ensure thread safe
- std::lock_guard<std::mutex> lock(mutable_string_config_lock);
+ std::unique_lock<std::shared_mutex> lock(mutable_string_config_lock);
UPDATE_FIELD(feild, value, std::string, need_persist);
}
return false;
@@ -457,4 +458,8 @@ std::pair<bool, std::string>
set_config(std::unordered_map<std::string, std::str
"failed, plz check config::custom_conf_path and
io status",
custom_conf_path)};
}
+
+std::shared_mutex* get_mutable_string_config_lock() {
+ return &mutable_string_config_lock;
+}
} // namespace doris::cloud::config
diff --git a/cloud/src/common/configbase.h b/cloud/src/common/configbase.h
index 6eee2639468..f1cc3716ba3 100644
--- a/cloud/src/common/configbase.h
+++ b/cloud/src/common/configbase.h
@@ -21,6 +21,7 @@
#include <functional>
#include <map>
#include <mutex>
+#include <shared_mutex>
#include <string>
#include <utility>
#include <vector>
@@ -178,4 +179,6 @@ bool init(const char* conf_file, bool fill_conf_map =
false, bool must_exist = t
std::pair<bool, std::string> set_config(std::unordered_map<std::string,
std::string> field_map,
bool need_persist, const std::string&
custom_conf_path);
+std::shared_mutex* get_mutable_string_config_lock();
+
} // namespace doris::cloud::config
diff --git a/cloud/src/meta-service/CMakeLists.txt
b/cloud/src/meta-service/CMakeLists.txt
index d11f87e7fa2..6a2ea4b921e 100644
--- a/cloud/src/meta-service/CMakeLists.txt
+++ b/cloud/src/meta-service/CMakeLists.txt
@@ -26,4 +26,5 @@ add_library(MetaService
mem_txn_kv.cpp
http_encode_key.cpp
txn_lazy_committer.cpp
+ delete_bitmap_lock_white_list.cpp
)
diff --git a/cloud/src/meta-service/delete_bitmap_lock_white_list.cpp
b/cloud/src/meta-service/delete_bitmap_lock_white_list.cpp
new file mode 100644
index 00000000000..6d2c00a085c
--- /dev/null
+++ b/cloud/src/meta-service/delete_bitmap_lock_white_list.cpp
@@ -0,0 +1,84 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "delete_bitmap_lock_white_list.h"
+
+namespace doris::cloud {
+
+bool use_new_version_random() {
+ std::mt19937 gen {std::random_device {}()};
+ auto p = 0.5;
+ std::bernoulli_distribution inject_fault {p};
+ if (inject_fault(gen)) {
+ return true;
+ }
+ return false;
+}
+
+DeleteBitmapLockWhiteList::DeleteBitmapLockWhiteList() {}
+
+DeleteBitmapLockWhiteList::~DeleteBitmapLockWhiteList() {}
+
+void DeleteBitmapLockWhiteList::init() {
+ update_white_list(config::delete_bitmap_lock_v2_white_list);
+}
+
+void DeleteBitmapLockWhiteList::update_white_list(std::string white_list) {
+ std::unique_lock<std::shared_mutex> lock(_rw_mutex);
+ _delete_bitmap_lock_v2_white_list_set.clear();
+ if (!white_list.empty()) {
+ auto white_list_vector = split(white_list, ';');
+ for (auto& item : white_list_vector) {
+ if (!item.empty()) {
+ trim(item);
+ _delete_bitmap_lock_v2_white_list_set.insert(item);
+ }
+ }
+ }
+ _last_white_list_value = white_list;
+}
+
+std::string
DeleteBitmapLockWhiteList::get_delete_bitmap_lock_version(std::string
instance_id) {
+ if (config::use_delete_bitmap_lock_random_version) {
+ if (use_new_version_random()) {
+ return "v2";
+ } else {
+ return "v1";
+ }
+ }
+ std::string white_list;
+ {
+ std::shared_lock<std::shared_mutex>
lock(*config::get_mutable_string_config_lock());
+ white_list = config::delete_bitmap_lock_v2_white_list;
+ }
+ if (white_list == "*") {
+ return "v2";
+ }
+ if (_last_white_list_value != white_list) {
+ update_white_list(white_list);
+ }
+ {
+ std::shared_lock<std::shared_mutex> lock(_rw_mutex);
+ auto it = _delete_bitmap_lock_v2_white_list_set.find(instance_id);
+ if (it != _delete_bitmap_lock_v2_white_list_set.end()) {
+ return "v2";
+ } else {
+ return "v1";
+ }
+ }
+}
+} // namespace doris::cloud
\ No newline at end of file
diff --git a/cloud/src/meta-service/delete_bitmap_lock_white_list.h
b/cloud/src/meta-service/delete_bitmap_lock_white_list.h
new file mode 100644
index 00000000000..87b5f06c9f8
--- /dev/null
+++ b/cloud/src/meta-service/delete_bitmap_lock_white_list.h
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <chrono>
+#include <random>
+#include <set>
+#include <shared_mutex>
+#include <string>
+#include <thread>
+
+#include "common/config.h"
+#include "common/logging.h"
+#include "common/string_util.h"
+
+namespace doris::cloud {
+class DeleteBitmapLockWhiteList;
+class DeleteBitmapLockWhiteList {
+public:
+ DeleteBitmapLockWhiteList();
+ ~DeleteBitmapLockWhiteList();
+ void init();
+ void update_white_list(std::string white_list);
+ std::string get_delete_bitmap_lock_version(std::string instance_id);
+
+private:
+ std::set<std::string> _delete_bitmap_lock_v2_white_list_set;
+ std::string _last_white_list_value;
+ std::shared_mutex _rw_mutex;
+};
+
+} // namespace doris::cloud
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index 2de89f26786..eaa93199e17 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -56,6 +56,7 @@
#include "cpp/sync_point.h"
#include "keys.h"
#include "meta-service/codec.h"
+#include "meta-service/delete_bitmap_lock_white_list.h"
#include "meta-service/doris_txn.h"
#include "meta-service/keys.h"
#include "meta-service/meta_service_helper.h"
@@ -77,6 +78,8 @@ MetaServiceImpl::MetaServiceImpl(std::shared_ptr<TxnKv>
txn_kv,
rate_limiter_ = rate_limiter;
rate_limiter_->init(this);
txn_lazy_committer_ = std::make_shared<TxnLazyCommitter>(txn_kv_);
+ delete_bitmap_lock_white_list_ =
std::make_shared<DeleteBitmapLockWhiteList>();
+ delete_bitmap_lock_white_list_->init();
}
MetaServiceImpl::~MetaServiceImpl() = default;
@@ -1984,14 +1987,10 @@ static bool check_delete_bitmap_lock(MetaServiceCode&
code, std::string& msg, st
}
return true;
}
-static bool use_new_version_random() {
- std::mt19937 gen {std::random_device {}()};
- auto p = 0.5;
- std::bernoulli_distribution inject_fault {p};
- if (inject_fault(gen)) {
- return true;
- }
- return false;
+
+void MetaServiceImpl::get_delete_bitmap_lock_version(std::string& use_version,
+ std::string& instance_id)
{
+ use_version =
delete_bitmap_lock_white_list_->get_delete_bitmap_lock_version(instance_id);
}
static bool process_pending_delete_bitmap(MetaServiceCode& code, std::string&
msg,
@@ -2135,10 +2134,6 @@ void
MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
const UpdateDeleteBitmapRequest*
request,
UpdateDeleteBitmapResponse*
response,
::google::protobuf::Closure* done) {
- std::string use_version = config::use_delete_bitmap_lock_version;
- if (config::use_delete_bitmap_lock_random_version &&
!use_new_version_random()) {
- use_version = "v1";
- }
RPC_PREPROCESS(update_delete_bitmap);
std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
if (cloud_unique_id.empty()) {
@@ -2154,6 +2149,8 @@ void
MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
LOG(WARNING) << msg << ", cloud_unique_id=" <<
request->cloud_unique_id();
return;
}
+ std::string use_version =
+
delete_bitmap_lock_white_list_->get_delete_bitmap_lock_version(instance_id);
RPC_RATE_LIMIT(update_delete_bitmap)
uint64_t fdb_txn_size = 0;
@@ -2332,7 +2329,8 @@ void
MetaServiceImpl::update_delete_bitmap(google::protobuf::RpcController* cont
<< " total_value_count=" << total_value_count << " unlock=" <<
unlock
<< " total_txn_put_keys=" << total_txn_put_keys
<< " total_txn_put_bytes=" << total_txn_put_bytes
- << " total_txn_size=" << total_txn_size << " total_txn_count="
<< total_txn_count;
+ << " total_txn_size=" << total_txn_size << " total_txn_count="
<< total_txn_count
+ << " instance_id=" << instance_id << " use_version=" <<
use_version;
}
void MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController*
controller,
@@ -2567,26 +2565,10 @@ static bool
put_delete_bitmap_update_lock_key(MetaServiceCode& code, std::string
void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
google::protobuf::RpcController* controller,
const GetDeleteBitmapUpdateLockRequest* request,
- GetDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done) {
+ GetDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done,
+ std::string& instance_id, MetaServiceCode& code, std::string& msg,
std::stringstream& ss) {
VLOG_DEBUG << "get delete bitmap update lock in v2 for table=" <<
request->table_id()
<< ",lock id=" << request->lock_id() << ",initiator=" <<
request->initiator();
- RPC_PREPROCESS(get_delete_bitmap_update_lock);
- std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
- if (cloud_unique_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "cloud unique id not set";
- return;
- }
-
- instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
- if (instance_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
- return;
- }
-
- RPC_RATE_LIMIT(get_delete_bitmap_update_lock)
auto table_id = request->table_id();
std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id,
table_id, -1});
bool first_retry = true;
@@ -2915,25 +2897,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
void MetaServiceImpl::get_delete_bitmap_update_lock_v1(
google::protobuf::RpcController* controller,
const GetDeleteBitmapUpdateLockRequest* request,
- GetDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done) {
+ GetDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done,
+ std::string& instance_id, MetaServiceCode& code, std::string& msg,
std::stringstream& ss) {
VLOG_DEBUG << "get delete bitmap update lock in v1 for table=" <<
request->table_id()
<< ",lock id=" << request->lock_id() << ",initiator=" <<
request->initiator();
- RPC_PREPROCESS(get_delete_bitmap_update_lock);
- std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
- if (cloud_unique_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "cloud unique id not set";
- return;
- }
- instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
- if (instance_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
- return;
- }
-
- RPC_RATE_LIMIT(get_delete_bitmap_update_lock)
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv_->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
@@ -3103,26 +3070,10 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v1(
void MetaServiceImpl::remove_delete_bitmap_update_lock_v2(
google::protobuf::RpcController* controller,
const RemoveDeleteBitmapUpdateLockRequest* request,
- RemoveDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done) {
+ RemoveDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done,
+ std::string& instance_id, MetaServiceCode& code, std::string& msg,
std::stringstream& ss) {
VLOG_DEBUG << "remove delete bitmap update lock in v2 for table=" <<
request->table_id()
<< ",lock id=" << request->lock_id() << ",initiator=" <<
request->initiator();
- RPC_PREPROCESS(remove_delete_bitmap_update_lock);
- std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
- if (cloud_unique_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "cloud unique id not set";
- return;
- }
-
- instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
- if (instance_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
- return;
- }
-
- RPC_RATE_LIMIT(remove_delete_bitmap_update_lock)
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv_->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
@@ -3198,24 +3149,10 @@ void
MetaServiceImpl::remove_delete_bitmap_update_lock_v2(
void MetaServiceImpl::remove_delete_bitmap_update_lock_v1(
google::protobuf::RpcController* controller,
const RemoveDeleteBitmapUpdateLockRequest* request,
- RemoveDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done) {
+ RemoveDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done,
+ std::string& instance_id, MetaServiceCode& code, std::string& msg,
std::stringstream& ss) {
VLOG_DEBUG << "remove delete bitmap update lock in v1 for table=" <<
request->table_id()
<< ",lock id=" << request->lock_id() << ",initiator=" <<
request->initiator();
- RPC_PREPROCESS(remove_delete_bitmap_update_lock);
- std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
- if (cloud_unique_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "cloud unique id not set";
- return;
- }
- instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
- if (instance_id.empty()) {
- code = MetaServiceCode::INVALID_ARGUMENT;
- msg = "empty instance_id";
- LOG(INFO) << msg << ", cloud_unique_id=" << cloud_unique_id;
- return;
- }
- RPC_RATE_LIMIT(remove_delete_bitmap_update_lock)
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv_->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
@@ -3279,14 +3216,30 @@ void
MetaServiceImpl::get_delete_bitmap_update_lock(google::protobuf::RpcControl
const
GetDeleteBitmapUpdateLockRequest* request,
GetDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done) {
- std::string use_version = config::use_delete_bitmap_lock_version;
- if (config::use_delete_bitmap_lock_random_version &&
!use_new_version_random()) {
- use_version = "v1";
+ RPC_PREPROCESS(get_delete_bitmap_update_lock);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ return;
}
+ RPC_RATE_LIMIT(get_delete_bitmap_update_lock)
+ std::string use_version =
+
delete_bitmap_lock_white_list_->get_delete_bitmap_lock_version(instance_id);
+ LOG(INFO) << "get_delete_bitmap_update_lock instance_id=" << instance_id
+ << " use_version=" << use_version;
if (use_version == "v2") {
- get_delete_bitmap_update_lock_v2(controller, request, response, done);
+ get_delete_bitmap_update_lock_v2(controller, request, response, done,
instance_id, code,
+ msg, ss);
} else {
- get_delete_bitmap_update_lock_v1(controller, request, response, done);
+ get_delete_bitmap_update_lock_v1(controller, request, response, done,
instance_id, code,
+ msg, ss);
}
}
@@ -3294,14 +3247,30 @@ void MetaServiceImpl::remove_delete_bitmap_update_lock(
google::protobuf::RpcController* controller,
const RemoveDeleteBitmapUpdateLockRequest* request,
RemoveDeleteBitmapUpdateLockResponse* response,
::google::protobuf::Closure* done) {
- std::string use_version = config::use_delete_bitmap_lock_version;
- if (config::use_delete_bitmap_lock_random_version &&
!use_new_version_random()) {
- use_version = "v1";
+ RPC_PREPROCESS(remove_delete_bitmap_update_lock);
+ std::string cloud_unique_id = request->has_cloud_unique_id() ?
request->cloud_unique_id() : "";
+ if (cloud_unique_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "cloud unique id not set";
+ return;
+ }
+ instance_id = get_instance_id(resource_mgr_, cloud_unique_id);
+ if (instance_id.empty()) {
+ code = MetaServiceCode::INVALID_ARGUMENT;
+ msg = "empty instance_id";
+ return;
}
+ RPC_RATE_LIMIT(remove_delete_bitmap_update_lock)
+ std::string use_version =
+
delete_bitmap_lock_white_list_->get_delete_bitmap_lock_version(instance_id);
+ LOG(INFO) << "remove_delete_bitmap_update_lock instance_id=" << instance_id
+ << " use_version=" << use_version;
if (use_version == "v2") {
- remove_delete_bitmap_update_lock_v2(controller, request, response,
done);
+ remove_delete_bitmap_update_lock_v2(controller, request, response,
done, instance_id, code,
+ msg, ss);
} else {
- remove_delete_bitmap_update_lock_v1(controller, request, response,
done);
+ remove_delete_bitmap_update_lock_v1(controller, request, response,
done, instance_id, code,
+ msg, ss);
}
}
diff --git a/cloud/src/meta-service/meta_service.h
b/cloud/src/meta-service/meta_service.h
index 0bc0b7ce8c1..b537c35b32d 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -29,6 +29,7 @@
#include "common/config.h"
#include "cpp/sync_point.h"
+#include "meta-service/delete_bitmap_lock_white_list.h"
#include "meta-service/txn_kv.h"
#include "meta-service/txn_lazy_committer.h"
#include "rate-limiter/rate_limiter.h"
@@ -324,6 +325,8 @@ public:
MetaServiceResponseStatus fix_tablet_stats(std::string cloud_unique_id_str,
std::string table_id_str);
+ void get_delete_bitmap_lock_version(std::string& use_version, std::string&
instance_id);
+
private:
std::pair<MetaServiceCode, std::string> alter_instance(
const AlterInstanceRequest* request,
@@ -332,27 +335,36 @@ private:
void get_delete_bitmap_update_lock_v2(google::protobuf::RpcController*
controller,
const
GetDeleteBitmapUpdateLockRequest* request,
GetDeleteBitmapUpdateLockResponse*
response,
- ::google::protobuf::Closure* done);
+ ::google::protobuf::Closure* done,
+ std::string& instance_id,
MetaServiceCode& code,
+ std::string& msg, std::stringstream&
ss);
void get_delete_bitmap_update_lock_v1(google::protobuf::RpcController*
controller,
const
GetDeleteBitmapUpdateLockRequest* request,
GetDeleteBitmapUpdateLockResponse*
response,
- ::google::protobuf::Closure* done);
+ ::google::protobuf::Closure* done,
+ std::string& instance_id,
MetaServiceCode& code,
+ std::string& msg, std::stringstream&
ss);
void remove_delete_bitmap_update_lock_v2(google::protobuf::RpcController*
controller,
const
RemoveDeleteBitmapUpdateLockRequest* request,
RemoveDeleteBitmapUpdateLockResponse* response,
- ::google::protobuf::Closure*
done);
+ ::google::protobuf::Closure* done,
+ std::string& instance_id,
MetaServiceCode& code,
+ std::string& msg,
std::stringstream& ss);
void remove_delete_bitmap_update_lock_v1(google::protobuf::RpcController*
controller,
const
RemoveDeleteBitmapUpdateLockRequest* request,
RemoveDeleteBitmapUpdateLockResponse* response,
- ::google::protobuf::Closure*
done);
+ ::google::protobuf::Closure* done,
+ std::string& instance_id,
MetaServiceCode& code,
+ std::string& msg,
std::stringstream& ss);
std::shared_ptr<TxnKv> txn_kv_;
std::shared_ptr<ResourceManager> resource_mgr_;
std::shared_ptr<RateLimiter> rate_limiter_;
std::shared_ptr<TxnLazyCommitter> txn_lazy_committer_;
+ std::shared_ptr<DeleteBitmapLockWhiteList> delete_bitmap_lock_white_list_;
};
class MetaServiceProxy final : public MetaService {
@@ -734,6 +746,10 @@ public:
call_impl(&cloud::MetaService::get_schema_dict, controller, request,
response, done);
}
+ void get_delete_bitmap_lock_version(std::string& use_version, std::string&
instance_id) {
+ impl_->get_delete_bitmap_lock_version(use_version, instance_id);
+ }
+
private:
template <typename Request, typename Response>
using MetaServiceMethod = void
(cloud::MetaService::*)(::google::protobuf::RpcController*,
diff --git a/cloud/src/meta-service/meta_service_job.cpp
b/cloud/src/meta-service/meta_service_job.cpp
index b00d834e322..0c77d5d33b8 100644
--- a/cloud/src/meta-service/meta_service_job.cpp
+++ b/cloud/src/meta-service/meta_service_job.cpp
@@ -672,21 +672,12 @@ static void
remove_delete_bitmap_update_lock(std::unique_ptr<Transaction>& txn,
}
}
-static bool use_new_version_random() {
- std::mt19937 gen {std::random_device {}()};
- auto p = 0.5;
- std::bernoulli_distribution inject_fault {p};
- if (inject_fault(gen)) {
- return true;
- }
- return false;
-}
-
void process_compaction_job(MetaServiceCode& code, std::string& msg,
std::stringstream& ss,
std::unique_ptr<Transaction>& txn,
const FinishTabletJobRequest* request,
FinishTabletJobResponse* response,
TabletJobInfoPB& recorded_job,
- std::string& instance_id, std::string& job_key,
bool& need_commit) {
+ std::string& instance_id, std::string& job_key,
bool& need_commit,
+ std::string& use_version) {
//==========================================================================
// check
//==========================================================================
@@ -757,10 +748,6 @@ void process_compaction_job(MetaServiceCode& code,
std::string& msg, std::string
INSTANCE_LOG(INFO) << "abort tablet compaction job, tablet_id=" <<
tablet_id
<< " key=" << hex(job_key);
if (compaction.has_delete_bitmap_lock_initiator()) {
- std::string use_version = config::use_delete_bitmap_lock_version;
- if (config::use_delete_bitmap_lock_random_version &&
!use_new_version_random()) {
- use_version = "v1";
- }
remove_delete_bitmap_update_lock(
txn, instance_id, table_id, tablet_id,
COMPACTION_DELETE_BITMAP_LOCK_ID,
compaction.delete_bitmap_lock_initiator(), use_version);
@@ -913,10 +900,6 @@ void process_compaction_job(MetaServiceCode& code,
std::string& msg, std::string
// remove delete bitmap update lock for MoW table
if (compaction.has_delete_bitmap_lock_initiator()) {
- std::string use_version = config::use_delete_bitmap_lock_version;
- if (config::use_delete_bitmap_lock_random_version &&
!use_new_version_random()) {
- use_version = "v1";
- }
bool success = check_and_remove_delete_bitmap_update_lock(
code, msg, ss, txn, instance_id, table_id, tablet_id,
COMPACTION_DELETE_BITMAP_LOCK_ID,
compaction.delete_bitmap_lock_initiator(),
@@ -1101,7 +1084,8 @@ void process_schema_change_job(MetaServiceCode& code,
std::string& msg, std::str
std::unique_ptr<Transaction>& txn,
const FinishTabletJobRequest* request,
FinishTabletJobResponse* response,
TabletJobInfoPB& recorded_job,
- std::string& instance_id, std::string& job_key,
bool& need_commit) {
+ std::string& instance_id, std::string& job_key,
bool& need_commit,
+ std::string& use_version) {
//==========================================================================
// check
//==========================================================================
@@ -1417,10 +1401,6 @@ void process_schema_change_job(MetaServiceCode& code,
std::string& msg, std::str
// process mow table, check lock
if (new_tablet_meta.enable_unique_key_merge_on_write()) {
- std::string use_version = config::use_delete_bitmap_lock_version;
- if (config::use_delete_bitmap_lock_random_version &&
!use_new_version_random()) {
- use_version = "v1";
- }
bool success = check_and_remove_delete_bitmap_update_lock(
code, msg, ss, txn, instance_id, new_table_id, new_tablet_id,
SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID,
schema_change.delete_bitmap_lock_initiator(),
@@ -1585,18 +1565,20 @@ void
MetaServiceImpl::finish_tablet_job(::google::protobuf::RpcController* contr
return;
}
});
-
+ std::string use_version =
+
delete_bitmap_lock_white_list_->get_delete_bitmap_lock_version(instance_id);
+ LOG(INFO) << "finish_tablet_job instance_id=" << instance_id << "
use_version=" << use_version;
// Process compaction commit
if (!request->job().compaction().empty()) {
process_compaction_job(code, msg, ss, txn, request, response,
recorded_job, instance_id,
- job_key, need_commit);
+ job_key, need_commit, use_version);
return;
}
// Process schema change commit
if (request->job().has_schema_change()) {
process_schema_change_job(code, msg, ss, txn, request, response,
recorded_job, instance_id,
- job_key, need_commit);
+ job_key, need_commit, use_version);
return;
}
}
diff --git a/cloud/test/meta_service_http_test.cpp
b/cloud/test/meta_service_http_test.cpp
index 7131505ae7f..c16e8e705df 100644
--- a/cloud/test/meta_service_http_test.cpp
+++ b/cloud/test/meta_service_http_test.cpp
@@ -1769,6 +1769,101 @@ TEST(MetaServiceHttpTest, UpdateConfig) {
ASSERT_EQ(retention_s, 259219);
}
}
+ {
+ auto [status_code, content] =
+ ctx.query<std::string>("update_config",
+
"configs=delete_bitmap_lock_v2_white_list="
+
"warehouse2;warehouse3&persist=true");
+
+ ASSERT_EQ(status_code, 200);
+ ASSERT_EQ(config::delete_bitmap_lock_v2_white_list,
"warehouse2;warehouse3");
+ auto& meta_service = ctx.meta_service_;
+ std::string use_version = "";
+ std::string instance_id = "warehouse1";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ instance_id = "warehouse2";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ instance_id = "warehouse3";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ config::Properties props;
+ ASSERT_TRUE(props.load(config::custom_conf_path.c_str(), true));
+ {
+ bool new_val_set = false;
+ std::string white_list = "";
+
ASSERT_TRUE(props.get_or_default("delete_bitmap_lock_v2_white_list", nullptr,
+ white_list, &new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_EQ(white_list, "warehouse2;warehouse3");
+ instance_id = "warehouse1";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ instance_id = "warehouse2";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ instance_id = "warehouse3";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ }
+ }
+ //resend config will rewrite it
+ {
+ auto [status_code, content] = ctx.query<std::string>(
+ "update_config",
"configs=delete_bitmap_lock_v2_white_list=''&persist=true");
+ ASSERT_EQ(status_code, 200);
+ ASSERT_EQ(config::delete_bitmap_lock_v2_white_list, "''");
+ auto& meta_service = ctx.meta_service_;
+ std::string use_version = "";
+ std::string instance_id = "warehouse1";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ instance_id = "warehouse2";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ instance_id = "warehouse3";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ }
+ {
+ auto [status_code, content] =
+ ctx.query<std::string>("update_config",
+
"configs=delete_bitmap_lock_v2_white_list="
+
"warehouse4;warehouse5&persist=true");
+ ASSERT_EQ(status_code, 200);
+ ASSERT_EQ(config::delete_bitmap_lock_v2_white_list,
"warehouse4;warehouse5");
+ auto& meta_service = ctx.meta_service_;
+ std::string use_version = "";
+ std::string instance_id = "warehouse3";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ instance_id = "warehouse4";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ instance_id = "warehouse5";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ config::Properties props;
+ ASSERT_TRUE(props.load(config::custom_conf_path.c_str(), true));
+ {
+ bool new_val_set = false;
+ std::string white_list = "";
+
ASSERT_TRUE(props.get_or_default("delete_bitmap_lock_v2_white_list", nullptr,
+ white_list, &new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_EQ(white_list, "warehouse4;warehouse5");
+ instance_id = "warehouse3";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ instance_id = "warehouse4";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ instance_id = "warehouse5";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ }
+ }
{
auto [status_code, content] = ctx.query<std::string>(
"update_config",
"configs=enable_s3_rate_limiter=false&persist=true");
@@ -1801,6 +1896,25 @@ TEST(MetaServiceHttpTest, UpdateConfig) {
ASSERT_TRUE(new_val_set);
ASSERT_FALSE(enable_s3_rate_limiter);
}
+ {
+ bool new_val_set = false;
+ std::string white_list = "";
+
ASSERT_TRUE(props.get_or_default("delete_bitmap_lock_v2_white_list", nullptr,
+ white_list, &new_val_set));
+ ASSERT_TRUE(new_val_set);
+ ASSERT_EQ(white_list, "warehouse4;warehouse5");
+ auto& meta_service = ctx.meta_service_;
+ std::string use_version = "";
+ std::string instance_id = "warehouse3";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v1");
+ instance_id = "warehouse4";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ instance_id = "warehouse5";
+ meta_service->get_delete_bitmap_lock_version(use_version,
instance_id);
+ ASSERT_EQ(use_version, "v2");
+ }
}
std::filesystem::remove(config::custom_conf_path);
config::custom_conf_path = original_conf_path;
diff --git a/cloud/test/meta_service_job_test.cpp
b/cloud/test/meta_service_job_test.cpp
index 048295d8643..f0e5cb8fe3d 100644
--- a/cloud/test/meta_service_job_test.cpp
+++ b/cloud/test/meta_service_job_test.cpp
@@ -152,11 +152,13 @@ void insert_rowsets(TxnKv* txn_kv, int64_t table_id,
int64_t index_id, int64_t p
}
MetaServiceCode get_delete_bitmap_lock(MetaServiceProxy* meta_service, int64_t
table_id,
- int64_t lock_id, int64_t initor,
int64_t expiration = 5) {
+ int64_t lock_id, int64_t initor,
+ std::string cloud_unique_id =
"test_cloud_unique_id",
+ int64_t expiration = 5) {
brpc::Controller cntl;
GetDeleteBitmapUpdateLockRequest req;
GetDeleteBitmapUpdateLockResponse res;
- req.set_cloud_unique_id("test_cloud_unique_id");
+ req.set_cloud_unique_id(cloud_unique_id);
req.set_table_id(table_id);
req.set_expiration(expiration);
req.set_lock_id(lock_id);
@@ -1115,6 +1117,41 @@ TEST(MetaServiceJobTest, CompactionJobTest) {
ASSERT_NO_FATAL_FAILURE(test_abort_compaction_job(1, 2, 3, 7));
}
+void check_delete_bitmap_lock(MetaServiceProxy* meta_service, std::string
instance_id,
+ int64_t table_id, int64_t lock_id, bool exist) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string lock_key = meta_delete_bitmap_update_lock_key({instance_id,
table_id, -1});
+ std::string lock_val;
+ DeleteBitmapUpdateLockPB lock_info;
+ TxnErrorCode err = txn->get(lock_key, &lock_val);
+ if (exist) {
+ ASSERT_TRUE(err == TxnErrorCode::TXN_OK);
+ } else {
+ ASSERT_TRUE(err == TxnErrorCode::TXN_KEY_NOT_FOUND);
+ return;
+ }
+ ASSERT_TRUE(lock_info.ParseFromString(lock_val));
+ ASSERT_TRUE(lock_info.lock_id() == lock_id);
+}
+
+void check_compaction_key(MetaServiceProxy* meta_service, std::string
instance_id, int64_t table_id,
+ int64_t initiator, bool exist) {
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
+ std::string tablet_compaction_key =
+ mow_tablet_compaction_key({instance_id, table_id, initiator});
+ std::string tablet_compaction_val;
+ TxnErrorCode err = txn->get(tablet_compaction_key, &tablet_compaction_val);
+ LOG(INFO) << "check_compaction_key instance_id=" << instance_id <<
",initiator=" << initiator
+ << ",exist=" << exist << ",err=" << err;
+ if (exist) {
+ ASSERT_TRUE(err == TxnErrorCode::TXN_OK);
+ } else {
+ ASSERT_TRUE(err == TxnErrorCode::TXN_KEY_NOT_FOUND);
+ }
+}
+
TEST(MetaServiceJobTest, DeleteBitmapUpdateLockCompatibilityTest) {
auto meta_service = get_meta_service();
auto sp = SyncPoint::get_instance();
@@ -1132,7 +1169,8 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
auto test_start_compaction_job = [&](int64_t table_id, int64_t index_id,
int64_t partition_id,
int64_t tablet_id,
TabletCompactionJobPB::CompactionType
type,
- std::string job_id = "job_id123") {
+ std::string job_id = "job_id123",
+ std::string instance_id =
"MetaServiceJobTest") {
StartTabletJobResponse res;
auto index_key = meta_tablet_idx_key({instance_id, tablet_id});
@@ -1160,7 +1198,8 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
int64_t tablet_id,
TabletCompactionJobPB::CompactionType type,
int64_t initiator = 12345,
- std::string job_id = "job_id123") {
+ std::string job_id = "job_id123",
+ std::string instance_id =
"MetaServiceJobTest") {
FinishTabletJobRequest req;
auto compaction = req.mutable_job()->add_compaction();
@@ -1318,21 +1357,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
// case 1: lock key does not exist, get and remove load lock in new way,
success
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
auto res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 123,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// case 2: lock key does not exist, get and remove load lock in old way,
success
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 123,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// case 3: lock key does not exist, get and remove compaction lock in new
way, success
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 12345);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1343,7 +1382,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
clear_rowsets(5);
// case 4: lock key does not exist, get and remove compaction lock in old
way, success
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 12345);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1358,14 +1397,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 5.2 load get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 222, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 5.3 compaction get lock in new way, failed
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 111);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
// 5.4 load remove lock in old way, success
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 222,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1374,14 +1413,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 6.2 load get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 222, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 6.3 compaction get lock in old way, failed
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 111);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
// 6.4 load remove lock in new way, success
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 222,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1391,14 +1430,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
clear_rowsets(table_id);
// 7.2 compaction get lock in new way
remove_delete_bitmap_lock(meta_service.get(), table_id);
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 777);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 7.3 load get lock in new way
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123, -1);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
// 7.4 compaction update delete bitmap in old way failed, abort compaction
job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
777);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1410,14 +1449,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 8.2 compaction get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 888);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 8.3 load get lock in old way
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 124, -1);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
// 8.4 compaction update delete bitmap in new way failed, abort compaction
job
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
888);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1429,11 +1468,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 9.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 901);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 9.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 902);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1443,13 +1482,13 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
clear_rowsets(5);
// 9.4 load get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 199, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 199,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 9.5 compaction3 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 903);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 9.6 compaction1 update delete bitmap in new way failed, abort job
@@ -1464,11 +1503,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 10.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1001);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 10.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1002);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1478,17 +1517,17 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
clear_rowsets(5);
// 10.4 load get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1910, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1910,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 10.5 compaction3 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1003);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 10.6 compaction1 update delete bitmap in new way success, commit job
failed
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1001);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1501,11 +1540,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 11.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1101);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 11.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1102);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1520,11 +1559,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 11.5 compaction3 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1103);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 11.6 compaction1 update delete bitmap in new way failed, abort job
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1101);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1536,11 +1575,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 12.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1201);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 12.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1202);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1550,17 +1589,17 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
clear_rowsets(5);
// 12.4 sc get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 12.5 compaction3 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1203);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 12.6 compaction1 update delete bitmap in new way success, commit job
failed
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1201);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1573,11 +1612,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 13.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1301);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 13.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1302);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1587,17 +1626,17 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
clear_rowsets(5);
// 13.4 load get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1390, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1390,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 13.5 compaction3 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1303);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 13.6 compaction1 update delete bitmap in new way failed, abort job
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1301);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1609,11 +1648,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 14.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1401);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 14.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1402);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1623,17 +1662,17 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
clear_rowsets(5);
// 14.4 load get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1490, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1490,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 14.5 compaction3 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1403);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 14.6 compaction1 update delete bitmap in old way failed, abort job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1401);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1645,11 +1684,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 15.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1501);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 15.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1502);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1659,17 +1698,17 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
clear_rowsets(5);
// 15.4 sc get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 15.5 compaction3 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1503);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 15.6 compaction1 update delete bitmap in old way failed, abort job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1501);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1681,11 +1720,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 16.2 compaction1 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1601);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 16.3 compaction2 get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1602);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
@@ -1695,17 +1734,17 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
clear_rowsets(5);
// 16.4 sc get and remove lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 16.5 compaction3 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1603);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 16.6 compaction1 update delete bitmap in old way failed, abort job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1601);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1717,21 +1756,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 17.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1701);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 17.3 load get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1793, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1793,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 17.4 compaction2 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1702);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 17.5 compaction1 update delete bitmap in new way failed, abort job
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1701);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1743,21 +1782,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 18.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1801);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 18.3 load get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1893, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1893,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 18.4 compaction2 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1802);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 18.5 compaction1 update delete bitmap in new way failed, abort job
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1801);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1769,21 +1808,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 19.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1901);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 19.3 sc get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 19.4 compaction2 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1902);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 19.5 compaction1 update delete bitmap in new way failed, abort job
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1901);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1795,21 +1834,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 20.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2001);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 20.3 sc get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 20.4 compaction2 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2002);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 20.5 compaction1 update delete bitmap in new way failed, abort job
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2001);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1821,21 +1860,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 21.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2101);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 21.3 load get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2190, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2190,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 21.4 compaction2 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2102);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 21.5 compaction1 update delete bitmap in old way failed, abort job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2101);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1847,21 +1886,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 22.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2201);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 22.3 load get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2290, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2290,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 22.4 compaction2 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2202);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 22.5 compaction1 update delete bitmap in old way failed, abort job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2201);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1873,21 +1912,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 23.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2301);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 23.3 sc get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2390, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2390,
-1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 23.4 compaction2 get lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2302);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 23.5 compaction1 update delete bitmap in old way failed, abort job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2301);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1899,21 +1938,21 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 24.2 compaction1 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2401);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 24.3 sc get and remove lock in new way
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 24.4 compaction2 get lock in old way
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2402);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 24.5 compaction1 update delete bitmap in old way failed, abort job
- config::use_delete_bitmap_lock_version = "v1";
+ config::delete_bitmap_lock_v2_white_list = "";
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2401);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
@@ -1945,7 +1984,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 2 compaction1 get lock in old/new way
- config::use_delete_bitmap_lock_version = lock_version;
+ if (lock_version == "v1") {
+ config::delete_bitmap_lock_v2_white_list = "";
+ } else {
+ config::delete_bitmap_lock_v2_white_list = "*";
+ }
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1,
2501);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 3 sc、load、compaction get and remove lock in new or old way
@@ -1955,7 +1998,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
for (int i = 0; i < 10; i++) {
int num = std::rand() % 3;
std::string use_version = (std::rand() % 2 == 0 ? "v2" : "v1");
- config::use_delete_bitmap_lock_version = use_version;
+ if (use_version == "v1") {
+ config::delete_bitmap_lock_v2_white_list = "";
+ } else {
+ config::delete_bitmap_lock_v2_white_list = "*";
+ }
LOG(INFO) << "i=" << i << ",num=" << num << ",use_version=" <<
use_version;
switch (num) {
case 0: {
@@ -2004,11 +2051,19 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
}
}
// 4 compaction2 get lock in old/new way
- config::use_delete_bitmap_lock_version = lock_version;
+ if (lock_version == "v1") {
+ config::delete_bitmap_lock_v2_white_list = "";
+ } else {
+ config::delete_bitmap_lock_v2_white_list = "*";
+ }
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1,
2502);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 5 if sc or load succeed，compaction1 remove lock in old/new way
failed
- config::use_delete_bitmap_lock_version = lock_version;
+ if (lock_version == "v1") {
+ config::delete_bitmap_lock_v2_white_list = "";
+ } else {
+ config::delete_bitmap_lock_v2_white_list = "*";
+ }
test_start_compaction_job(table_id, 2, 3, 5,
TabletCompactionJobPB::BASE);
test_commit_compaction_job(table_id, 2, 3, 5,
TabletCompactionJobPB::BASE, 2501);
if (load_or_sc_succeed) {
@@ -2016,6 +2071,596 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
}
clear_rowsets(5);
}
+
+ //white list test
+ //1. normal test
+ config::delete_bitmap_lock_v2_white_list =
+
"instance_id1;instance_id1;instance_id2;仓库6;Instance_id7;Instance_id*;instance_id10$;"
+ "instance_id11 ;;instance_id12";
+ //after parse config, the following instance will use v2:
+ //instance_id1,instance_id2,instance_id11,instance_id12
+ table_id = 7890;
+ //compaction
+ remove_delete_bitmap_lock(meta_service.get(), table_id);
+ std::string instance_id_x = "instance_id1";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 101);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 101,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
101);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
101, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 101,
false);
+ clear_rowsets(5);
+
+ instance_id_x = "instance_id2";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 102);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 102,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
102);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
102, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 102,
false);
+ clear_rowsets(5);
+
+ instance_id_x = "instance_id3";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 103);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
103);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
103, "job_id123",
+ instance_id_x);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
+ clear_rowsets(5);
+
+ instance_id_x = "instance_id6";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 106);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
106);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
106, "job_id123",
+ instance_id_x);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
+ clear_rowsets(5);
+
+ instance_id_x = "instance_id7";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 107);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
107);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
107, "job_id123",
+ instance_id_x);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
+ clear_rowsets(5);
+
+ instance_id_x = "instance_id10";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1010);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1010);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1010, "job_id123",
+ instance_id_x);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
+ clear_rowsets(5);
+
+ instance_id_x = "instance_id11";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1011);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 1011,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1011);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1011, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 1011,
false);
+ clear_rowsets(5);
+
+ instance_id_x = "instance_id12";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1012);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 1012,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1012);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1012, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 1012,
false);
+ clear_rowsets(5);
+
+ //load
+ instance_id_x = "instance_id1";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1121, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1121, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1121,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1121, false);
+
+ instance_id_x = "instance_id2";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1122, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1122, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1122,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1122, false);
+
+ instance_id_x = "instance_id3";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1123, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1123, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1123,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1123, false);
+
+ instance_id_x = "instance_id6";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1126, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1126, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1126,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1126, false);
+
+ instance_id_x = "instance_id7";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1127, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1127, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1127,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1127, false);
+
+ instance_id_x = "instance_id10";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1130, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1130, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1130,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1130, false);
+
+ instance_id_x = "instance_id11";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1131, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1131, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1131,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1131, false);
+
+ instance_id_x = "instance_id12";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1132, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1132, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 1132,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
1132, false);
+
+ //sc
+ instance_id_x = "instance_id1";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1301);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1301);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ instance_id_x = "instance_id2";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1302);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1302);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ instance_id_x = "instance_id3";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1303);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1303);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ instance_id_x = "instance_id6";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1306);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1306);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ instance_id_x = "instance_id7";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1307);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1307);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ instance_id_x = "instance_id10";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1310);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1310);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ instance_id_x = "instance_id11";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1311);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1311);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ instance_id_x = "instance_id12";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 1312);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1312);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ //2. empty white list
+ config::delete_bitmap_lock_v2_white_list = "";
+ // compaction
+ instance_id_x = "instance_id20";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ //compaction
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
2000, "job_id123",
+ instance_id_x);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
+ // load
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 2001, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
2001, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 2001,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
2001, false);
+ // sc
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 2002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
2002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ //3. 1000 item white list
+ std::stringstream ss;
+ ss << "instance_id0;";
+ for (int i = 1; i < 1000; i++) {
+ ss << "instance_id" << i << ";";
+ }
+ ss << "instance_id1000";
+ config::delete_bitmap_lock_v2_white_list = ss.str();
+ instance_id_x = "instance_id0";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ //compaction
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 3000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 3000,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
3000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
3000, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 3000,
false);
+ clear_rowsets(5);
+ // load
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 3001, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
3001, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 3001,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
3001, false);
+ // sc
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 3002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
3002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ //4.config single item
+ config::delete_bitmap_lock_v2_white_list = "instance_id40";
+ instance_id_x = "instance_id40";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+
+ //instance_id40->v2
+ //compaction
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 4000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 4000,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
4000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
4000, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 4000,
false);
+ clear_rowsets(5);
+ // load
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 4001, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
4001, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 4001,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
4001, false);
+ // sc
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 4002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
4002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ //instance_id41->v1
+ instance_id_x = "instance_id41";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+ //compaction
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 4100);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
4100);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
4100, "job_id123",
+ instance_id_x);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
+ // load
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 4101, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
4101, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 4101,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
4101, false);
+ // sc
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 4102);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
4102);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ //5.config *
+ //instance_id50->v2
+ config::delete_bitmap_lock_v2_white_list = "*";
+ instance_id_x = "instance_id50";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+
+ //compaction
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 5000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 5000,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
5000);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
5000, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 5000,
false);
+ clear_rowsets(5);
+ // load
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 5001, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
5001, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 5001,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
5001, false);
+ // sc
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 5002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
5002);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+
+ //instance_id51->v2
+ instance_id_x = "instance_id51";
+ sp->set_call_back("get_instance_id", [&](auto&& args) {
+ auto* ret = try_any_cast_ret<std::string>(args);
+ ret->first = instance_id_x;
+ ret->second = true;
+ });
+
+ //compaction
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 5100);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 5100,
true);
+ test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
+ instance_id_x);
+ res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
5100);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
5100, "job_id123",
+ instance_id_x);
+ check_compaction_key(meta_service.get(), instance_id_x, table_id, 5100,
false);
+ clear_rowsets(5);
+ // load
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 5101, -1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
5101, true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 5101,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id,
5101, false);
+ // sc
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, 5102);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
+ res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
5102);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
}
TEST(MetaServiceJobTest, CompactionJobWithMoWTest) {
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index 81da3646772..0278b504315 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -3171,7 +3171,7 @@ void remove_delete_bitmap_lock(MetaServiceProxy*
meta_service, int64_t table_id)
TEST(CheckerTest, check_compaction_key) {
config::enable_mow_compaction_key_check = true;
config::compaction_key_check_expiration_diff_seconds = 0;
- config::use_delete_bitmap_lock_version = "v2";
+ config::delete_bitmap_lock_v2_white_list = "*";
std::string instance_id = "test_check_compaction_key";
[[maybe_unused]] auto sp = SyncPoint::get_instance();
std::unique_ptr<int, std::function<void(int*)>> defer(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
