This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 7518da40b21 branch-3.1: [improve](mow) refactor mow update lock for
schema change to reduce txn conflict #51301 (#52916)
7518da40b21 is described below
commit 7518da40b21b0a43c82c3b8e17b9adc0e74c3a21
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Jul 8 17:46:40 2025 +0800
branch-3.1: [improve](mow) refactor mow update lock for schema change to
reduce txn conflict #51301 (#52916)
Cherry-picked from #51301
Co-authored-by: meiyi <[email protected]>
---
cloud/src/common/config.h | 4 +-
cloud/src/meta-service/keys.cpp | 16 +-
cloud/src/meta-service/keys.h | 8 +-
cloud/src/meta-service/meta_service.cpp | 163 ++++-----
cloud/src/meta-service/meta_service.h | 5 +
cloud/src/meta-service/meta_service_job.cpp | 42 +--
cloud/src/recycler/checker.cpp | 32 +-
cloud/src/recycler/checker.h | 2 +-
cloud/src/recycler/recycler.cpp | 15 +-
cloud/test/meta_service_job_test.cpp | 256 +++++++++-----
cloud/test/meta_service_test.cpp | 141 ++++++--
cloud/test/recycler_test.cpp | 42 +--
gensrc/proto/cloud.proto | 2 +-
...t_unique_schema_key_change_multi_tablets.groovy | 391 +++++++++++++++++++++
14 files changed, 818 insertions(+), 301 deletions(-)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 883af8f64da..f435af3a3bb 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -118,8 +118,8 @@ CONF_mInt64(recycle_task_threshold_seconds, "10800"); // 3h
// **just for TEST**
CONF_Bool(force_immediate_recycle, "false");
-CONF_mBool(enable_mow_compaction_key_check, "false");
-CONF_mInt64(compaction_key_check_expiration_diff_seconds, "600"); // 10min
+CONF_mBool(enable_mow_job_key_check, "false");
+CONF_mInt64(mow_job_key_check_expiration_diff_seconds, "600"); // 10min
CONF_String(test_s3_ak, "");
CONF_String(test_s3_sk, "");
diff --git a/cloud/src/meta-service/keys.cpp b/cloud/src/meta-service/keys.cpp
index b00988b7a77..d4e14bacbe5 100644
--- a/cloud/src/meta-service/keys.cpp
+++ b/cloud/src/meta-service/keys.cpp
@@ -51,7 +51,7 @@ static const char* META_KEY_INFIX_SCHEMA =
"schema";
static const char* META_KEY_INFIX_DELETE_BITMAP = "delete_bitmap";
static const char* META_KEY_INFIX_DELETE_BITMAP_LOCK = "delete_bitmap_lock";
static const char* META_KEY_INFIX_DELETE_BITMAP_PENDING =
"delete_bitmap_pending";
-static const char* META_KEY_INFIX_MOW_TABLET_COMPACTION = "mow_tablet_comp";
+static const char* META_KEY_INFIX_MOW_TABLET_JOB = "mow_tablet_job";
static const char* META_KEY_INFIX_SCHEMA_DICTIONARY =
"tablet_schema_pb_dict";
static const char* RECYCLE_KEY_INFIX_INDEX = "index";
@@ -117,7 +117,7 @@ static void encode_prefix(const T& t, std::string* key) {
StatsTabletKeyInfo, TableVersionKeyInfo,
JobTabletKeyInfo, JobRecycleKeyInfo, RLJobProgressKeyInfo,
CopyJobKeyInfo, CopyFileKeyInfo, StorageVaultKeyInfo,
MetaSchemaPBDictionaryInfo,
- MowTabletCompactionInfo>);
+ MowTabletJobInfo>);
key->push_back(CLOUD_USER_KEY_SPACE01);
// Prefixes for key families
@@ -137,7 +137,7 @@ static void encode_prefix(const T& t, std::string* key) {
|| std::is_same_v<T, MetaDeleteBitmapInfo>
|| std::is_same_v<T, MetaDeleteBitmapUpdateLockInfo>
|| std::is_same_v<T, MetaPendingDeleteBitmapInfo>
- || std::is_same_v<T, MowTabletCompactionInfo>) {
+ || std::is_same_v<T, MowTabletJobInfo>) {
encode_bytes(META_KEY_PREFIX, key);
} else if constexpr (std::is_same_v<T, PartitionVersionKeyInfo>
|| std::is_same_v<T, TableVersionKeyInfo>) {
@@ -302,11 +302,11 @@ void meta_delete_bitmap_update_lock_key(const
MetaDeleteBitmapUpdateLockInfo& in
encode_int64(std::get<2>(in), out); // partition_id
}
-void mow_tablet_compaction_key(const MowTabletCompactionInfo& in, std::string*
out) {
- encode_prefix(in, out); // 0x01 "meta"
${instance_id}
- encode_bytes(META_KEY_INFIX_MOW_TABLET_COMPACTION, out); //
"mow_tablet_comp"
- encode_int64(std::get<1>(in), out); // table_id
- encode_int64(std::get<2>(in), out); // initiator
+void mow_tablet_job_key(const MowTabletJobInfo& in, std::string* out) {
+ encode_prefix(in, out); // 0x01 "meta"
${instance_id}
+ encode_bytes(META_KEY_INFIX_MOW_TABLET_JOB, out); // "mow_tablet_job"
+ encode_int64(std::get<1>(in), out); // table_id
+ encode_int64(std::get<2>(in), out); // initiator
}
void meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in,
std::string* out) {
diff --git a/cloud/src/meta-service/keys.h b/cloud/src/meta-service/keys.h
index b819d8e2608..05e97a0a6a5 100644
--- a/cloud/src/meta-service/keys.h
+++ b/cloud/src/meta-service/keys.h
@@ -45,7 +45,7 @@
// 0x01 "meta" ${instance_id} "delete_bitmap_pending" ${table_id}
-> PendingDeleteBitmapPB
// 0x01 "meta" ${instance_id} "delete_bitmap" ${tablet_id} ${rowset_id}
${version} ${segment_id} -> roaringbitmap
// 0x01 "meta" ${instance_id} "tablet_schema_pb_dict" ${index_id}
-> SchemaCloudDictionary
-// 0x01 "meta" ${instance_id} "mow_tablet_comp" ${table_id} ${initiator_id}
-> MowTabletCompactionPB
+// 0x01 "meta" ${instance_id} "mow_tablet_job" ${table_id} ${initiator_id}
-> MowTabletJobPB
//
// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} -> TabletStatsPB
// 0x01 "stats" ${instance_id} "tablet" ${table_id} ${index_id}
${partition_id} ${tablet_id} "data_size" -> int64
@@ -192,7 +192,7 @@ using TableVersionKeyInfo = BasicKeyInfo<27,
std::tuple<std::string, int64_t, in
// 0:instance_id
1:index_id
using MetaSchemaPBDictionaryInfo = BasicKeyInfo<28 , std::tuple<std::string,
int64_t>>;
// 0:instance_id
1:table_id 2:initiator
-using MowTabletCompactionInfo = BasicKeyInfo<29 , std::tuple<std::string,
int64_t, int64_t>>;
+using MowTabletJobInfo = BasicKeyInfo<29 , std::tuple<std::string, int64_t,
int64_t>>;
void instance_key(const InstanceKeyInfo& in, std::string* out);
static inline std::string instance_key(const InstanceKeyInfo& in) {
std::string s; instance_key(in, &s); return s; }
@@ -226,7 +226,7 @@ void meta_delete_bitmap_key(const MetaDeleteBitmapInfo& in,
std::string* out);
void meta_delete_bitmap_update_lock_key(const MetaDeleteBitmapUpdateLockInfo&
in, std::string* out);
void meta_pending_delete_bitmap_key(const MetaPendingDeleteBitmapInfo& in,
std::string* out);
void meta_schema_pb_dictionary_key(const MetaSchemaPBDictionaryInfo& in,
std::string* out);
-void mow_tablet_compaction_key(const MowTabletCompactionInfo& in, std::string*
out);
+void mow_tablet_job_key(const MowTabletJobInfo& in, std::string* out);
static inline std::string meta_rowset_key(const MetaRowsetKeyInfo& in) {
std::string s; meta_rowset_key(in, &s); return s; }
static inline std::string meta_rowset_tmp_key(const MetaRowsetTmpKeyInfo& in)
{ std::string s; meta_rowset_tmp_key(in, &s); return s; }
static inline std::string meta_tablet_idx_key(const MetaTabletIdxKeyInfo& in)
{ std::string s; meta_tablet_idx_key(in, &s); return s; }
@@ -236,7 +236,7 @@ static inline std::string meta_delete_bitmap_key(const
MetaDeleteBitmapInfo& in)
static inline std::string meta_delete_bitmap_update_lock_key(const
MetaDeleteBitmapUpdateLockInfo& in) { std::string s;
meta_delete_bitmap_update_lock_key(in, &s); return s; }
static inline std::string meta_pending_delete_bitmap_key(const
MetaPendingDeleteBitmapInfo& in) { std::string s;
meta_pending_delete_bitmap_key(in, &s); return s; }
static inline std::string meta_schema_pb_dictionary_key(const
MetaSchemaPBDictionaryInfo& in) { std::string s;
meta_schema_pb_dictionary_key(in, &s); return s; }
-static inline std::string mow_tablet_compaction_key(const
MowTabletCompactionInfo& in) { std::string s; mow_tablet_compaction_key(in,
&s); return s; }
+static inline std::string mow_tablet_job_key(const MowTabletJobInfo& in) {
std::string s; mow_tablet_job_key(in, &s); return s; }
std::string recycle_key_prefix(std::string_view instance_id);
void recycle_index_key(const RecycleIndexKeyInfo& in, std::string* out);
diff --git a/cloud/src/meta-service/meta_service.cpp
b/cloud/src/meta-service/meta_service.cpp
index fd01639fcff..72ced9b8d10 100644
--- a/cloud/src/meta-service/meta_service.cpp
+++ b/cloud/src/meta-service/meta_service.cpp
@@ -1966,20 +1966,19 @@ static bool check_delete_bitmap_lock(MetaServiceCode&
code, std::string& msg, st
code = MetaServiceCode::LOCK_EXPIRED;
return false;
}
- if (use_version == "v2" && lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
- std::string tablet_compaction_key =
- mow_tablet_compaction_key({instance_id, table_id,
lock_initiator});
- std::string tablet_compaction_val;
- err = txn->get(tablet_compaction_key, &tablet_compaction_val);
+ if (use_version == "v2" && is_job_delete_bitmap_lock_id(lock_id)) {
+ std::string tablet_job_key = mow_tablet_job_key({instance_id,
table_id, lock_initiator});
+ std::string tablet_job_val;
+ err = txn->get(tablet_job_key, &tablet_job_val);
if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
- ss << "tablet compaction key not found, table_id=" << table_id <<
" lock_id" << lock_id
+ ss << "tablet job key not found, table_id=" << table_id << "
lock_id=" << lock_id
<< " initiator=" << lock_initiator;
msg = ss.str();
code = MetaServiceCode::LOCK_EXPIRED;
return false;
}
if (err != TxnErrorCode::TXN_OK) {
- ss << "failed to get tablet compaction info, err=" << err;
+ ss << "failed to get mow tablet job info, err=" << err;
msg = ss.str();
code = cast_as<ErrCategory::READ>(err);
return false;
@@ -2636,25 +2635,23 @@ void
MetaServiceImpl::get_delete_bitmap(google::protobuf::RpcController* control
}
}
-static bool put_mow_tablet_compaction_key(MetaServiceCode& code, std::string&
msg,
- std::unique_ptr<Transaction>& txn,
- std::string& instance_id, int64_t
table_id,
- int64_t lock_id, int64_t initiator,
int64_t expiration,
- std::string& current_lock_msg) {
- std::string tablet_compaction_key =
- mow_tablet_compaction_key({instance_id, table_id, initiator});
- std::string tablet_compaction_val;
- MowTabletCompactionPB mow_tablet_compaction;
- mow_tablet_compaction.set_expiration(expiration);
- mow_tablet_compaction.SerializeToString(&tablet_compaction_val);
- if (tablet_compaction_val.empty()) {
+static bool put_mow_tablet_job_key(MetaServiceCode& code, std::string& msg,
+ std::unique_ptr<Transaction>& txn,
std::string& instance_id,
+ int64_t table_id, int64_t lock_id, int64_t
initiator,
+ int64_t expiration, std::string&
current_lock_msg) {
+ std::string tablet_job_key = mow_tablet_job_key({instance_id, table_id,
initiator});
+ std::string tablet_job_val;
+ MowTabletJobPB mow_tablet_job;
+ mow_tablet_job.set_expiration(expiration);
+ mow_tablet_job.SerializeToString(&tablet_job_val);
+ if (tablet_job_val.empty()) {
code = MetaServiceCode::PROTOBUF_SERIALIZE_ERR;
- msg = "MowTabletCompactionPB serialization error";
+ msg = "MowTabletJobPB serialization error";
return false;
}
- txn->put(tablet_compaction_key, tablet_compaction_val);
- LOG(INFO) << "xxx put tablet compaction key=" << hex(tablet_compaction_key)
- << " table_id=" << table_id << " lock_id=" << lock_id << "
initiator=" << initiator
+ txn->put(tablet_job_key, tablet_job_val);
+ LOG(INFO) << "xxx put tablet job key=" << hex(tablet_job_key) << "
table_id=" << table_id
+ << " lock_id=" << lock_id << " initiator=" << initiator
<< " expiration=" << expiration << ", " << current_lock_msg;
return true;
}
@@ -2927,22 +2924,20 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
lock_info.set_lock_id(request->lock_id());
// compaction does not use this expiration, only used when upgrade
ms
lock_info.set_expiration(expiration);
- if (request->lock_id() != COMPACTION_DELETE_BITMAP_LOCK_ID) {
+ if (!is_job_delete_bitmap_lock_id(request->lock_id())) {
lock_info.add_initiators(request->initiator());
} else {
// in normal case, this should remove 0 kvs
// but when upgrade ms, if there are ms with old and new
versions, it works
- std::string tablet_compaction_key_begin =
- mow_tablet_compaction_key({instance_id, table_id, 0});
- std::string tablet_compaction_key_end =
- mow_tablet_compaction_key({instance_id, table_id,
INT64_MAX});
- txn->remove(tablet_compaction_key_begin,
tablet_compaction_key_end);
- LOG(INFO) << "remove mow tablet compaction kv, begin="
- << hex(tablet_compaction_key_begin)
- << " end=" << hex(tablet_compaction_key_end) << "
table_id=" << table_id;
- if (!put_mow_tablet_compaction_key(code, msg, txn,
instance_id, table_id,
- request->lock_id(),
request->initiator(),
- expiration,
current_lock_msg)) {
+ std::string tablet_job_key_begin =
mow_tablet_job_key({instance_id, table_id, 0});
+ std::string tablet_job_key_end =
+ mow_tablet_job_key({instance_id, table_id, INT64_MAX});
+ txn->remove(tablet_job_key_begin, tablet_job_key_end);
+ LOG(INFO) << "remove mow tablet job kv, begin=" <<
hex(tablet_job_key_begin)
+ << " end=" << hex(tablet_job_key_end) << "
table_id=" << table_id;
+ if (!put_mow_tablet_job_key(code, msg, txn, instance_id,
table_id,
+ request->lock_id(),
request->initiator(), expiration,
+ current_lock_msg)) {
return;
}
}
@@ -2957,7 +2952,7 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
msg = "failed to parse DeleteBitmapUpdateLockPB";
return;
}
- if (lock_info.lock_id() != COMPACTION_DELETE_BITMAP_LOCK_ID) {
+ if (!is_job_delete_bitmap_lock_id(lock_info.lock_id())) {
if (lock_info.expiration() > 0 && lock_info.expiration() <
now) {
LOG(INFO) << "delete bitmap lock expired, continue to
process. lock_id="
<< lock_info.lock_id() << " table_id=" <<
table_id
@@ -2978,7 +2973,7 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
lock_info.set_lock_id(request->lock_id());
// compaction does not use the expiration, only used when
upgrade ms
lock_info.set_expiration(expiration);
- if (request->lock_id() != COMPACTION_DELETE_BITMAP_LOCK_ID) {
+ if (!is_job_delete_bitmap_lock_id(request->lock_id())) {
bool found = false;
for (auto initiator : lock_info.initiators()) {
if (request->initiator() == initiator) {
@@ -2993,18 +2988,16 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
lock_key_not_found = true;
// in normal case, this should remove 0 kvs
// but when upgrade ms, if there are ms with old and new
versions, it works
- std::string tablet_compaction_key_begin =
- mow_tablet_compaction_key({instance_id, table_id,
0});
- std::string tablet_compaction_key_end =
- mow_tablet_compaction_key({instance_id, table_id,
INT64_MAX});
- txn->remove(tablet_compaction_key_begin,
tablet_compaction_key_end);
- LOG(INFO) << "remove mow tablet compaction kv, begin="
- << hex(tablet_compaction_key_begin)
- << " end=" << hex(tablet_compaction_key_end)
- << " table_id=" << table_id;
- if (!put_mow_tablet_compaction_key(code, msg, txn,
instance_id, table_id,
- request->lock_id(),
request->initiator(),
- expiration,
current_lock_msg)) {
+ std::string tablet_job_key_begin =
+ mow_tablet_job_key({instance_id, table_id, 0});
+ std::string tablet_job_key_end =
+ mow_tablet_job_key({instance_id, table_id,
INT64_MAX});
+ txn->remove(tablet_job_key_begin, tablet_job_key_end);
+ LOG(INFO) << "remove mow tablet job kv, begin=" <<
hex(tablet_job_key_begin)
+ << " end=" << hex(tablet_job_key_end) << "
table_id=" << table_id;
+ if (!put_mow_tablet_job_key(code, msg, txn, instance_id,
table_id,
+ request->lock_id(),
request->initiator(),
+ expiration, current_lock_msg))
{
return;
}
}
@@ -3014,28 +3007,28 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
return;
}
} else {
- if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) {
- std::string current_lock_msg = "locked by lock_id=-1";
- if (!put_mow_tablet_compaction_key(code, msg, txn,
instance_id, table_id,
- request->lock_id(),
request->initiator(),
- expiration,
current_lock_msg)) {
+ if (request->lock_id() == lock_info.lock_id()) {
+ std::string current_lock_msg =
+ "locked by lock_id=" +
std::to_string(lock_info.lock_id());
+ if (!put_mow_tablet_job_key(code, msg, txn, instance_id,
table_id,
+ request->lock_id(),
request->initiator(),
+ expiration, current_lock_msg))
{
return;
}
} else {
// check if compaction key is expired
bool has_unexpired_compaction = false;
int64_t unexpired_expiration = 0;
- std::string key0 = mow_tablet_compaction_key({instance_id,
table_id, 0});
- std::string key1 = mow_tablet_compaction_key({instance_id,
table_id + 1, 0});
- MowTabletCompactionPB mow_tablet_compaction;
+ std::string key0 = mow_tablet_job_key({instance_id,
table_id, 0});
+ std::string key1 = mow_tablet_job_key({instance_id,
table_id + 1, 0});
+ MowTabletJobPB mow_tablet_job;
std::unique_ptr<RangeGetIterator> it;
- int64_t expired_compaction_num = 0;
+ int64_t expired_job_num = 0;
do {
err = txn->get(key0, key1, &it);
if (err != TxnErrorCode::TXN_OK) {
code = cast_as<ErrCategory::READ>(err);
- ss << "internal error, failed to get mow tablet
compaction, err="
- << err;
+ ss << "internal error, failed to get mow tablet
job, err=" << err;
msg = ss.str();
LOG(WARNING) << msg;
return;
@@ -3043,23 +3036,22 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
while (it->has_next() && !has_unexpired_compaction) {
auto [k, v] = it->next();
- if
(!mow_tablet_compaction.ParseFromArray(v.data(), v.size()))
- [[unlikely]] {
+ if (!mow_tablet_job.ParseFromArray(v.data(),
v.size())) [[unlikely]] {
code = MetaServiceCode::PROTOBUF_PARSE_ERR;
- msg = "failed to parse MowTabletCompactionPB";
+ msg = "failed to parse MowTabletJobPB";
return;
}
- if (mow_tablet_compaction.expiration() > 0 &&
- mow_tablet_compaction.expiration() < now) {
- LOG(INFO) << "remove mow tablet compaction
lock. table_id="
- << table_id << " lock_id=" <<
lock_info.lock_id()
- << " expiration=" <<
mow_tablet_compaction.expiration()
+ if (mow_tablet_job.expiration() > 0 &&
+ mow_tablet_job.expiration() < now) {
+ LOG(INFO) << "remove mow tablet job lock.
table_id=" << table_id
+ << " lock_id=" << lock_info.lock_id()
+ << " expiration=" <<
mow_tablet_job.expiration()
<< " now=" << now << " key=" <<
hex(k);
txn->remove(k);
- expired_compaction_num++;
+ expired_job_num++;
} else {
has_unexpired_compaction = true;
- unexpired_expiration =
mow_tablet_compaction.expiration();
+ unexpired_expiration =
mow_tablet_job.expiration();
}
}
key0 = it->next_begin_key(); // Update to next
smallest key for iteration
@@ -3074,13 +3066,22 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
code = MetaServiceCode::LOCK_CONFLICT;
return;
}
- // all compaction is expired
+ // all job is expired
lock_info.set_lock_id(request->lock_id());
lock_info.set_expiration(expiration);
lock_info.clear_initiators();
- lock_info.add_initiators(request->initiator());
std::string current_lock_msg =
- std::to_string(expired_compaction_num) + "
compaction is expired";
+ std::to_string(expired_job_num) + " job is
expired";
+ if (!is_job_delete_bitmap_lock_id(request->lock_id())) {
+ lock_info.add_initiators(request->initiator());
+ } else {
+ lock_key_not_found = true;
+ if (!put_mow_tablet_job_key(code, msg, txn,
instance_id, table_id,
+ request->lock_id(),
request->initiator(),
+ expiration,
current_lock_msg)) {
+ return;
+ }
+ }
if (!put_delete_bitmap_update_lock_key(code, msg, txn,
table_id,
request->lock_id(),
request->initiator(),
lock_key,
lock_info, current_lock_msg)) {
@@ -3092,11 +3093,11 @@ void MetaServiceImpl::get_delete_bitmap_update_lock_v2(
err = txn->commit();
TEST_SYNC_POINT_CALLBACK("get_delete_bitmap_update_lock:commit:conflict",
&first_retry,
- &err);
+ request, &err);
if (err == TxnErrorCode::TXN_OK) {
break;
} else if (err == TxnErrorCode::TXN_CONFLICT && lock_key_not_found &&
- request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID &&
+ is_job_delete_bitmap_lock_id(request->lock_id()) &&
config::delete_bitmap_enable_retry_txn_conflict &&
first_retry) {
// if err is TXN_CONFLICT, and the lock id is -1, do a fast retry
if (err == TxnErrorCode::TXN_CONFLICT) {
@@ -3238,13 +3239,13 @@ void
MetaServiceImpl::remove_delete_bitmap_update_lock_v2(
stats.put_counter += txn->num_put_keys();
stats.del_counter += txn->num_del_keys();
};
- if (request->lock_id() == COMPACTION_DELETE_BITMAP_LOCK_ID) {
- std::string tablet_compaction_key =
- mow_tablet_compaction_key({instance_id, request->table_id(),
request->initiator()});
- txn->remove(tablet_compaction_key);
- LOG(INFO) << "remove tablet compaction lock, table_id=" <<
request->table_id()
+ if (is_job_delete_bitmap_lock_id(request->lock_id())) {
+ std::string tablet_job_key =
+ mow_tablet_job_key({instance_id, request->table_id(),
request->initiator()});
+ txn->remove(tablet_job_key);
+ LOG(INFO) << "remove tablet job lock, table_id=" << request->table_id()
<< " lock_id=" << request->lock_id() << " initiator=" <<
request->initiator()
- << " key=" << hex(tablet_compaction_key);
+ << " key=" << hex(tablet_job_key);
} else {
std::string lock_key =
meta_delete_bitmap_update_lock_key({instance_id,
request->table_id(), -1});
diff --git a/cloud/src/meta-service/meta_service.h
b/cloud/src/meta-service/meta_service.h
index 45c9a486d08..1c054af6776 100644
--- a/cloud/src/meta-service/meta_service.h
+++ b/cloud/src/meta-service/meta_service.h
@@ -57,6 +57,11 @@ static void* run_bthread_work(void* arg) {
return nullptr;
}
+[[maybe_unused]] inline static bool is_job_delete_bitmap_lock_id(int64_t
lock_id) {
+ return lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID ||
+ lock_id == SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID;
+}
+
class MetaServiceImpl : public cloud::MetaService {
public:
MetaServiceImpl(std::shared_ptr<TxnKv> txn_kv,
std::shared_ptr<ResourceManager> resource_mgr,
diff --git a/cloud/src/meta-service/meta_service_job.cpp
b/cloud/src/meta-service/meta_service_job.cpp
index c6791aa4346..10dcfe1ed75 100644
--- a/cloud/src/meta-service/meta_service_job.cpp
+++ b/cloud/src/meta-service/meta_service_job.cpp
@@ -508,7 +508,7 @@ static bool check_and_remove_delete_bitmap_update_lock(
code = MetaServiceCode::LOCK_EXPIRED;
return false;
}
- if (use_version == "v2" && lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
+ if (use_version == "v2" && is_job_delete_bitmap_lock_id(lock_id)) {
// when upgrade ms, prevent old ms get delete bitmap update lock
if (lock_info.initiators_size() > 0) {
ss << "compaction lock has " << lock_info.initiators_size() << "
initiators";
@@ -516,29 +516,27 @@ static bool check_and_remove_delete_bitmap_update_lock(
code = MetaServiceCode::LOCK_EXPIRED;
return false;
}
- std::string tablet_compaction_key =
- mow_tablet_compaction_key({instance_id, table_id,
lock_initiator});
- std::string tablet_compaction_val;
- err = txn->get(tablet_compaction_key, &tablet_compaction_val);
+ std::string tablet_job_key = mow_tablet_job_key({instance_id,
table_id, lock_initiator});
+ std::string tablet_job_val;
+ err = txn->get(tablet_job_key, &tablet_job_val);
if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
ss << "lock initiator " << lock_initiator << " not exist";
msg = ss.str();
code = MetaServiceCode::LOCK_EXPIRED;
return false;
} else if (err != TxnErrorCode::TXN_OK) {
- ss << "failed to get tablet compaction key, instance_id=" <<
instance_id
+ ss << "failed to get tablet job key, instance_id=" << instance_id
<< " table_id=" << table_id << " tablet_id=" << tablet_id
- << " initiator=" << lock_initiator << " key=" <<
hex(tablet_compaction_key)
+ << " initiator=" << lock_initiator << " key=" <<
hex(tablet_job_key)
<< " err=" << err;
msg = ss.str();
code = cast_as<ErrCategory::READ>(err);
return false;
}
- txn->remove(tablet_compaction_key);
- INSTANCE_LOG(INFO) << "remove tablet compaction lock, table_id=" <<
table_id
+ txn->remove(tablet_job_key);
+ INSTANCE_LOG(INFO) << "remove tablet job lock, table_id=" << table_id
<< " tablet_id=" << tablet_id << " lock_id=" <<
lock_id
- << " initiator=" << lock_initiator
- << " key=" << hex(tablet_compaction_key);
+ << " initiator=" << lock_initiator << " key=" <<
hex(tablet_job_key);
// may left a lock key for -1
return true;
} else {
@@ -645,24 +643,22 @@ static void
remove_delete_bitmap_update_lock(std::unique_ptr<Transaction>& txn,
VLOG_DEBUG << "remove_delete_bitmap_update_lock table_id=" << table_id
<< " initiator=" << lock_initiator << " tablet_id=" << tablet_id
<< " lock_id=" << lock_id << " use_version=" << use_version;
- if (use_version == "v2" && lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID) {
- std::string tablet_compaction_key =
- mow_tablet_compaction_key({instance_id, table_id,
lock_initiator});
- std::string tablet_compaction_val;
- TxnErrorCode err = txn->get(tablet_compaction_key,
&tablet_compaction_val);
+ if (use_version == "v2" && is_job_delete_bitmap_lock_id(lock_id)) {
+ std::string tablet_job_key = mow_tablet_job_key({instance_id,
table_id, lock_initiator});
+ std::string tablet_job_val;
+ TxnErrorCode err = txn->get(tablet_job_key, &tablet_job_val);
if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
remove_delete_bitmap_update_lock_v1(txn, instance_id, table_id,
tablet_id, lock_id,
lock_initiator);
} else if (err != TxnErrorCode::TXN_OK) {
- INSTANCE_LOG(WARNING) << "failed to get tablet compaction key,
instance_id="
- << instance_id << " table_id=" << table_id
- << " initiator=" << lock_initiator
- << " key=" << hex(tablet_compaction_key) <<
" err=" << err;
+ INSTANCE_LOG(WARNING) << "failed to get tablet job key,
instance_id=" << instance_id
+ << " table_id=" << table_id << " initiator="
<< lock_initiator
+ << " key=" << hex(tablet_job_key) << " err="
<< err;
return;
} else {
- txn->remove(tablet_compaction_key);
- INSTANCE_LOG(INFO) << "remove tablet compaction key, table_id=" <<
table_id
- << ", key=" << hex(tablet_compaction_key)
+ txn->remove(tablet_job_key);
+ INSTANCE_LOG(INFO) << "remove tablet job key, table_id=" <<
table_id
+ << ", key=" << hex(tablet_job_key)
<< " initiator=" << lock_initiator;
}
} else {
diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp
index 801b4f3a3be..5b09b581ee6 100644
--- a/cloud/src/recycler/checker.cpp
+++ b/cloud/src/recycler/checker.cpp
@@ -195,8 +195,8 @@ int Checker::start() {
}
}
- if (config::enable_mow_compaction_key_check) {
- if (int ret = checker->do_mow_compaction_key_check(); ret !=
0) {
+ if (config::enable_mow_job_key_check) {
+ if (int ret = checker->do_mow_job_key_check(); ret != 0) {
success = false;
}
}
@@ -1573,11 +1573,11 @@ int
InstanceChecker::do_delete_bitmap_storage_optimize_check(int version) {
return (failed_tablets_num > 0) ? 1 : 0;
}
-int InstanceChecker::do_mow_compaction_key_check() {
+int InstanceChecker::do_mow_job_key_check() {
std::unique_ptr<RangeGetIterator> it;
- std::string begin = mow_tablet_compaction_key({instance_id_, 0, 0});
- std::string end = mow_tablet_compaction_key({instance_id_, INT64_MAX, 0});
- MowTabletCompactionPB mow_tablet_compaction;
+ std::string begin = mow_tablet_job_key({instance_id_, 0, 0});
+ std::string end = mow_tablet_job_key({instance_id_, INT64_MAX, 0});
+ MowTabletJobPB mow_tablet_job;
do {
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv_->create_txn(&txn);
@@ -1587,7 +1587,7 @@ int InstanceChecker::do_mow_compaction_key_check() {
}
err = txn->get(begin, end, &it);
if (err != TxnErrorCode::TXN_OK) {
- LOG(WARNING) << "failed to get mow tablet compaction key, err=" <<
err;
+ LOG(WARNING) << "failed to get mow tablet job key, err=" << err;
return -1;
}
int64_t now = duration_cast<std::chrono::seconds>(
@@ -1599,18 +1599,18 @@ int InstanceChecker::do_mow_compaction_key_check() {
k1.remove_prefix(1);
std::vector<std::tuple<std::variant<int64_t, std::string>, int,
int>> out;
decode_key(&k1, &out);
- // 0x01 "meta" ${instance_id} "mow_tablet_comp" ${table_id}
${initiator}
+ // 0x01 "meta" ${instance_id} "mow_tablet_job" ${table_id}
${initiator}
auto table_id = std::get<int64_t>(std::get<0>(out[3]));
auto initiator = std::get<int64_t>(std::get<0>(out[4]));
- if (!mow_tablet_compaction.ParseFromArray(v.data(), v.size()))
[[unlikely]] {
- LOG(WARNING) << "failed to parse MowTabletCompactionPB";
+ if (!mow_tablet_job.ParseFromArray(v.data(), v.size()))
[[unlikely]] {
+ LOG(WARNING) << "failed to parse MowTabletJobPB";
return -1;
}
- int64_t expiration = mow_tablet_compaction.expiration();
- //check compaction key failed should meet both following two
condition:
- //1.compaction key is expired
- //2.table lock key is not found or key is not expired
- if (expiration < now -
config::compaction_key_check_expiration_diff_seconds) {
+ int64_t expiration = mow_tablet_job.expiration();
+ // check job key failed should meet both following two condition:
+ // 1. job key is expired
+ // 2. table lock key is not found or key is not expired
+ if (expiration < now -
config::mow_job_key_check_expiration_diff_seconds) {
std::string lock_key =
meta_delete_bitmap_update_lock_key({instance_id_,
table_id, -1});
std::string lock_val;
@@ -1632,7 +1632,7 @@ int InstanceChecker::do_mow_compaction_key_check() {
}
if (reason != "") {
LOG(WARNING) << fmt::format(
- "[compaction key check fails] compaction key check
fail for "
+ "[compaction key check fails] mow job key check
fail for "
"instance_id={}, table_id={}, initiator={},
expiration={}, now={}, "
"reason={}",
instance_id_, table_id, initiator, expiration,
now, reason);
diff --git a/cloud/src/recycler/checker.h b/cloud/src/recycler/checker.h
index 6c3929848da..d1b909a4203 100644
--- a/cloud/src/recycler/checker.h
+++ b/cloud/src/recycler/checker.h
@@ -105,7 +105,7 @@ public:
// version = 2 : https://github.com/apache/doris/pull/49822
int do_delete_bitmap_storage_optimize_check(int version = 1);
- int do_mow_compaction_key_check();
+ int do_mow_job_key_check();
// If there are multiple buckets, return the minimum lifecycle; if there
are no buckets (i.e.
// all accessors are HdfsAccessor), return INT64_MAX.
diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp
index b7238efbdc6..7282ee4fad5 100644
--- a/cloud/src/recycler/recycler.cpp
+++ b/cloud/src/recycler/recycler.cpp
@@ -1358,18 +1358,15 @@ int InstanceRecycler::recycle_versions() {
auto tbl_version_key = table_version_key({instance_id_, db_id,
table_id});
txn->remove(tbl_version_key);
LOG(WARNING) << "remove table version kv " << hex(tbl_version_key);
- // 3. Remove mow delete bitmap update lock and tablet compaction lock
+ // 3. Remove mow delete bitmap update lock and tablet job lock
std::string lock_key =
meta_delete_bitmap_update_lock_key({instance_id_, table_id, -1});
txn->remove(lock_key);
LOG(WARNING) << "remove delete bitmap update lock kv " <<
hex(lock_key);
- std::string tablet_compaction_key_begin =
- mow_tablet_compaction_key({instance_id_, table_id, 0});
- std::string tablet_compaction_key_end =
- mow_tablet_compaction_key({instance_id_, table_id, INT64_MAX});
- txn->remove(tablet_compaction_key_begin, tablet_compaction_key_end);
- LOG(WARNING) << "remove mow tablet compaction kv, begin="
- << hex(tablet_compaction_key_begin)
- << " end=" << hex(tablet_compaction_key_end) << " db_id="
<< db_id
+ std::string tablet_job_key_begin = mow_tablet_job_key({instance_id_,
table_id, 0});
+ std::string tablet_job_key_end = mow_tablet_job_key({instance_id_,
table_id, INT64_MAX});
+ txn->remove(tablet_job_key_begin, tablet_job_key_end);
+ LOG(WARNING) << "remove mow tablet job kv, begin=" <<
hex(tablet_job_key_begin)
+ << " end=" << hex(tablet_job_key_end) << " db_id=" <<
db_id
<< " table_id=" << table_id;
err = txn->commit();
if (err != TxnErrorCode::TXN_OK) {
diff --git a/cloud/test/meta_service_job_test.cpp
b/cloud/test/meta_service_job_test.cpp
index caa2db13c41..b772d1c2211 100644
--- a/cloud/test/meta_service_job_test.cpp
+++ b/cloud/test/meta_service_job_test.cpp
@@ -153,9 +153,8 @@ void insert_rowsets(TxnKv* txn_kv, int64_t table_id,
int64_t index_id, int64_t p
}
MetaServiceCode get_delete_bitmap_lock(MetaServiceProxy* meta_service, int64_t
table_id,
- int64_t lock_id, int64_t initor,
- std::string cloud_unique_id =
"test_cloud_unique_id",
- int64_t expiration = 5) {
+ int64_t lock_id, int64_t initor,
int64_t expiration = 5,
+ std::string cloud_unique_id =
"test_cloud_unique_id") {
brpc::Controller cntl;
GetDeleteBitmapUpdateLockRequest req;
GetDeleteBitmapUpdateLockResponse res;
@@ -213,10 +212,9 @@ void remove_delete_bitmap_lock(MetaServiceProxy*
meta_service, int64_t table_id)
std::unique_ptr<Transaction> txn;
ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
txn->remove(lock_key);
- std::string tablet_compaction_key_begin =
mow_tablet_compaction_key({instance_id, table_id, 0});
- std::string tablet_compaction_key_end =
- mow_tablet_compaction_key({instance_id, table_id, INT64_MAX});
- txn->remove(tablet_compaction_key_begin, tablet_compaction_key_end);
+ std::string tablet_job_key_begin = mow_tablet_job_key({instance_id,
table_id, 0});
+ std::string tablet_job_key_end = mow_tablet_job_key({instance_id,
table_id, INT64_MAX});
+ txn->remove(tablet_job_key_begin, tablet_job_key_end);
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
}
@@ -277,11 +275,12 @@ void start_schema_change_job(MetaServiceProxy*
meta_service, int64_t table_id, i
EXPECT_EQ(job_pb.schema_change().id(), job_id) << ' ' << initiator;
};
-void finish_schema_change_job(
- MetaService* meta_service, int64_t tablet_id, int64_t new_tablet_id,
- const std::string& job_id, const std::string& initiator,
- const std::vector<doris::RowsetMetaCloudPB>& output_rowsets,
FinishTabletJobResponse& res,
- FinishTabletJobRequest_Action action = FinishTabletJobRequest::COMMIT)
{
+void finish_schema_change_job(MetaService* meta_service, int64_t tablet_id,
int64_t new_tablet_id,
+ const std::string& job_id, const std::string&
initiator,
+ const std::vector<doris::RowsetMetaCloudPB>&
output_rowsets,
+ FinishTabletJobResponse& res,
+ FinishTabletJobRequest_Action action =
FinishTabletJobRequest::COMMIT,
+ int64_t delete_bitmap_lock_initiator = 12345) {
brpc::Controller cntl;
FinishTabletJobRequest req;
req.set_action(action);
@@ -307,9 +306,18 @@ void finish_schema_change_job(
}
sc->set_id(job_id);
sc->set_initiator(initiator);
- sc->set_delete_bitmap_lock_initiator(12345);
+ sc->set_delete_bitmap_lock_initiator(delete_bitmap_lock_initiator);
meta_service->finish_tablet_job(&cntl, &req, &res, nullptr);
}
+
+void finish_schema_change_job(MetaService* meta_service, int64_t tablet_id,
int64_t new_tablet_id,
+ const std::string& job_id, const std::string&
initiator,
+ const std::vector<doris::RowsetMetaCloudPB>&
output_rowsets,
+ FinishTabletJobResponse& res, int64_t
delete_bitmap_lock_initiator) {
+ finish_schema_change_job(meta_service, tablet_id, new_tablet_id, job_id,
initiator,
+ output_rowsets, res,
FinishTabletJobRequest::COMMIT,
+ delete_bitmap_lock_initiator);
+}
} // namespace
TEST(MetaServiceJobTest, StartCompactionArguments) {
@@ -1133,24 +1141,24 @@ void check_delete_bitmap_lock(MetaServiceProxy*
meta_service, std::string instan
DeleteBitmapUpdateLockPB lock_info;
TxnErrorCode err = txn->get(lock_key, &lock_val);
if (exist) {
- ASSERT_TRUE(err == TxnErrorCode::TXN_OK);
+ ASSERT_EQ(err, TxnErrorCode::TXN_OK) << ", lock_id=" << lock_id << ",
exist=" << exist;
} else {
- ASSERT_TRUE(err == TxnErrorCode::TXN_KEY_NOT_FOUND);
+ ASSERT_EQ(err, TxnErrorCode::TXN_KEY_NOT_FOUND)
+ << ", lock_id=" << lock_id << ", exist=" << exist;
return;
}
ASSERT_TRUE(lock_info.ParseFromString(lock_val));
- ASSERT_TRUE(lock_info.lock_id() == lock_id);
+ ASSERT_EQ(lock_info.lock_id(), lock_id) << ", exist=" << exist;
}
-void check_compaction_key(MetaServiceProxy* meta_service, std::string
instance_id, int64_t table_id,
- int64_t initiator, bool exist) {
+void check_job_key(MetaServiceProxy* meta_service, std::string instance_id,
int64_t table_id,
+ int64_t initiator, bool exist) {
std::unique_ptr<Transaction> txn;
ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
- std::string tablet_compaction_key =
- mow_tablet_compaction_key({instance_id, table_id, initiator});
- std::string tablet_compaction_val;
- TxnErrorCode err = txn->get(tablet_compaction_key, &tablet_compaction_val);
- LOG(INFO) << "check_compaction_key instance_id=" << instance_id <<
",initiator=" << initiator
+ std::string tablet_job_key = mow_tablet_job_key({instance_id, table_id,
initiator});
+ std::string tablet_job_val;
+ TxnErrorCode err = txn->get(tablet_job_key, &tablet_job_val);
+ LOG(INFO) << "check_job_key instance_id=" << instance_id << ",initiator="
<< initiator
<< ",exist=" << exist << ",err=" << err;
if (exist) {
ASSERT_TRUE(err == TxnErrorCode::TXN_OK);
@@ -1364,6 +1372,73 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
int64_t table_id = 111;
remove_delete_bitmap_lock(meta_service.get(), table_id);
+ // test compaction and schema change
+ std::set<std::tuple<int, int, int, int, MetaServiceCode>> use_versions = {
+ {1, 1, 1, 1, MetaServiceCode::OK},
+ {1, 1, 1, 2, MetaServiceCode::OK},
+ {1, 1, 2, 1, MetaServiceCode::OK},
+ {1, 1, 2, 2, MetaServiceCode::OK},
+ {1, 2, 1, 1, MetaServiceCode::LOCK_CONFLICT},
+ {1, 2, 1, 2, MetaServiceCode::LOCK_CONFLICT},
+ {1, 2, 2, 1, MetaServiceCode::OK},
+ {1, 2, 2, 2, MetaServiceCode::OK},
+ {2, 1, 1, 1, MetaServiceCode::OK},
+ {2, 1, 1, 2, MetaServiceCode::OK},
+ {2, 1, 2, 1, MetaServiceCode::OK},
+ {2, 1, 2, 2, MetaServiceCode::OK},
+ {2, 2, 1, 1, MetaServiceCode::LOCK_CONFLICT},
+ {2, 2, 1, 2, MetaServiceCode::LOCK_CONFLICT},
+ {2, 2, 2, 1, MetaServiceCode::OK},
+ {2, 2, 2, 2, MetaServiceCode::OK}};
+ for (auto& use_version : use_versions) {
+ int64_t version1 = std::get<0>(use_version);
+ int64_t version2 = std::get<1>(use_version);
+ int64_t version3 = std::get<2>(use_version);
+ int64_t version4 = std::get<3>(use_version);
+ LOG(INFO) << "version1=" << version1 << ", version2=" << version2
+ << ", version3=" << version3 << ", version4=" << version4;
+
+ config::delete_bitmap_lock_v2_white_list = version1 == 1 ? "" : "*";
+ // 1. compaction1 get lock, but not commit job, the lock is timeout
+ auto tablet_id = 5;
+ auto new_tablet_id = 6;
+ test_start_compaction_job(table_id, 2, 3, tablet_id,
TabletCompactionJobPB::BASE);
+ auto res_code = get_delete_bitmap_lock(meta_service.get(), table_id,
-1, 12345, 1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ sleep(2);
+ // 2. schema change1 get lock (commit; or timeout)
+ config::delete_bitmap_lock_v2_white_list = version2 == 1 ? "" : "*";
+ ASSERT_NO_FATAL_FAILURE(
+ create_tablet(meta_service.get(), table_id, 2, 3,
new_tablet_id, true, true));
+ StartTabletJobResponse start_sc_res;
+ ASSERT_NO_FATAL_FAILURE(start_schema_change_job(meta_service.get(),
table_id, 2, 3,
+ tablet_id,
new_tablet_id, "job1", "be1",
+ start_sc_res));
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2,
12346);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ std::vector<doris::RowsetMetaCloudPB> output_rowsets;
+ for (int64_t i = 0; i < 5; ++i) {
+ output_rowsets.push_back(create_rowset(new_tablet_id, i + 2, i +
2));
+ CreateRowsetResponse create_rowset_response;
+ commit_rowset(meta_service.get(), output_rowsets.back(),
create_rowset_response);
+ ASSERT_EQ(create_rowset_response.status().code(),
MetaServiceCode::OK) << i;
+ }
+ FinishTabletJobResponse finish_sc_res;
+ finish_schema_change_job(meta_service.get(), tablet_id, new_tablet_id,
"job1", "be1",
+ output_rowsets, finish_sc_res, 12346);
+ ASSERT_EQ(finish_sc_res.status().code(), MetaServiceCode::OK);
+ // 3. compaction2 get lock, compaction1 start commit
+ config::delete_bitmap_lock_v2_white_list = version3 == 1 ? "" : "*";
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1,
12347);
+ ASSERT_EQ(res_code, std::get<4>(use_version));
+ config::delete_bitmap_lock_v2_white_list = version4 == 1 ? "" : "*";
+ test_commit_compaction_job(table_id, 2, 3, tablet_id,
TabletCompactionJobPB::BASE);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
+ clear_rowsets(table_id);
+ clear_rowsets(new_tablet_id);
+ remove_delete_bitmap_lock(meta_service.get(), table_id);
+ }
+
// case 1: lock key does not exist, get and remove load lock in new way,
success
config::delete_bitmap_lock_v2_white_list = "*";
auto res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123,
-1);
@@ -1387,7 +1462,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// case 4: lock key does not exist, get and remove compaction lock in old
way, success
config::delete_bitmap_lock_v2_white_list = "";
@@ -1398,7 +1472,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// case 5:
// 5.1 lock key does not exist
@@ -1437,7 +1510,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
remove_delete_bitmap_lock(meta_service.get(), table_id);
clear_rowsets(table_id);
// 7.2 compaction get lock in new way
- remove_delete_bitmap_lock(meta_service.get(), table_id);
config::delete_bitmap_lock_v2_white_list = "*";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 777);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1450,7 +1522,13 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
777);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 777);
- clear_rowsets(5);
+ // 7.5 load get lock fail
+ std::string white_lists[] = {"", "*"};
+ for (auto& white_list : white_lists) {
+ config::delete_bitmap_lock_v2_white_list = white_list;
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::LOCK_CONFLICT);
+ }
// case 8:
// 8.1 lock key does not exist
@@ -1469,7 +1547,12 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
888);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 888);
- clear_rowsets(5);
+ // 8.5 load get lock success
+ for (auto& white_list : white_lists) {
+ config::delete_bitmap_lock_v2_white_list = white_list;
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 123,
-1);
+ ASSERT_EQ(res_code, MetaServiceCode::OK);
+ }
// case 9:
// 9.1 lock key does not exist
@@ -1488,7 +1571,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
902);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 9.4 load get and remove lock in old way
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 199, -1);
@@ -1504,7 +1586,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
901);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 901);
- clear_rowsets(5);
// case 10:
// 10.1 lock key does not exist
@@ -1523,7 +1604,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1002);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 10.4 load get and remove lock in old way
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1910, -1);
@@ -1541,7 +1621,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1001);
ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
- clear_rowsets(5);
// case 11:
// 11.1 lock key does not exist
@@ -1560,7 +1639,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1102);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 11.4 sc get and remove lock in old way
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1576,7 +1654,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1101);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1101);
- clear_rowsets(5);
// case 12:
// 12.1 lock key does not exist
@@ -1595,7 +1672,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1202);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 12.4 sc get and remove lock in old way
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
@@ -1613,7 +1689,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1201);
ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
- clear_rowsets(5);
// case 13:
// 13.1 lock key does not exist
@@ -1632,7 +1707,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1302);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 13.4 load get and remove lock in old way
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1390, -1);
@@ -1649,7 +1723,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1301);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1301);
- clear_rowsets(5);
// case 14:
// 14.1 lock key does not exist
@@ -1668,7 +1741,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1402);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 14.4 load get and remove lock in old way
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 1490, -1);
@@ -1685,7 +1757,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1401);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1401);
- clear_rowsets(5);
// case 15:
// 15.1 lock key does not exist
@@ -1704,7 +1775,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1502);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 15.4 sc get and remove lock in old way
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
@@ -1721,7 +1791,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1501);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1501);
- clear_rowsets(5);
// case 16:
// 16.1 lock key does not exist
@@ -1740,7 +1809,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1602);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- clear_rowsets(5);
// 16.4 sc get and remove lock in old way
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
@@ -1757,7 +1825,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1601);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1601);
- clear_rowsets(5);
// case 17:
// 17.1 lock key does not exist
@@ -1783,7 +1850,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1701);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1701);
- clear_rowsets(5);
// case 18:
// 18.1 lock key does not exist
@@ -1809,7 +1875,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1801);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1801);
- clear_rowsets(5);
// case 19:
// 19.1 lock key does not exist
@@ -1835,7 +1900,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1901);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 1901);
- clear_rowsets(5);
// case 20:
// 20.1 lock key does not exist
@@ -1847,11 +1911,12 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 20.3 sc get and remove lock in new way
config::delete_bitmap_lock_v2_white_list = "*";
- res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1, 1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 20.4 compaction2 get lock in old way
+ sleep(2);
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2002);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1861,7 +1926,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2001);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 2001);
- clear_rowsets(5);
// case 21:
// 21.1 lock key does not exist
@@ -1887,7 +1951,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2101);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 2101);
- clear_rowsets(5);
// case 22:
// 22.1 lock key does not exist
@@ -1913,7 +1976,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2201);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 2201);
- clear_rowsets(5);
// case 23:
// 23.1 lock key does not exist
@@ -1939,7 +2001,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2301);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 2301);
- clear_rowsets(5);
// case 24:
// 24.1 lock key does not exist
@@ -1951,11 +2012,12 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 24.3 sc get and remove lock in new way
config::delete_bitmap_lock_v2_white_list = "*";
- res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
+ res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -2, -1, 1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
// 24.4 compaction2 get lock in old way
+ sleep(2);
config::delete_bitmap_lock_v2_white_list = "";
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 2402);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -1965,7 +2027,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
2401);
ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
test_abort_compaction_job(table_id, 2, 3, 5, 2401);
- clear_rowsets(5);
// fuzzy case 1:
// 1.1 lock key does not exist
@@ -2026,7 +2087,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
break;
}
case 1: {
- res_code = get_delete_bitmap_lock(meta_service.get(),
table_id, -2, -1);
+ res_code = get_delete_bitmap_lock(meta_service.get(),
table_id, -2, -1, 1);
if (res_code == MetaServiceCode::OK) {
res_code = remove_delete_bitmap_lock(meta_service.get(),
table_id, -2, -1);
if (res_code == MetaServiceCode::OK) {
@@ -2035,6 +2096,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
}
LOG(INFO) << "i=" << i << ",load_or_sc_succeed=" <<
load_or_sc_succeed
<< ",compaction_succeed=" << compaction_succeed;
+ sleep(2);
break;
}
case 2: {
@@ -2050,7 +2112,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
compaction_succeed = true;
}
}
- clear_rowsets(5);
}
LOG(INFO) << "i=" << i << ",load_or_sc_succeed=" <<
load_or_sc_succeed
<< ",compaction_succeed=" << compaction_succeed;
@@ -2077,7 +2138,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
if (load_or_sc_succeed) {
ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_EXPIRED);
}
- clear_rowsets(5);
}
//white list test
@@ -2099,15 +2159,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 101);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 101,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 101, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
101);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
101, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 101,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 101, false);
instance_id_x = "instance_id2";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2118,15 +2177,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 102);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 102,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 102, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
102);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
102, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 102,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 102, false);
instance_id_x = "instance_id3";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2144,7 +2202,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
103, "job_id123",
instance_id_x);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
- clear_rowsets(5);
instance_id_x = "instance_id6";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2162,7 +2219,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
106, "job_id123",
instance_id_x);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
- clear_rowsets(5);
instance_id_x = "instance_id7";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2180,7 +2236,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
107, "job_id123",
instance_id_x);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
- clear_rowsets(5);
instance_id_x = "instance_id10";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2198,7 +2253,6 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1010, "job_id123",
instance_id_x);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
false);
- clear_rowsets(5);
instance_id_x = "instance_id11";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2209,15 +2263,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1011);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 1011,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 1011, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1011);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1011, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 1011,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 1011, false);
instance_id_x = "instance_id12";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2228,15 +2281,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 1012);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 1012,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 1012, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
1012);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
1012, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 1012,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 1012, false);
//load
instance_id_x = "instance_id1";
@@ -2355,7 +2407,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1301);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
instance_id_x = "instance_id2";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2368,7 +2420,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1302);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
instance_id_x = "instance_id3";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2433,7 +2485,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1311);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
instance_id_x = "instance_id12";
sp->set_call_back("get_instance_id", [&](auto&& args) {
@@ -2446,7 +2498,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
1312);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
//2. empty white list
config::delete_bitmap_lock_v2_white_list = "";
@@ -2501,15 +2553,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 3000);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 3000,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 3000, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
3000);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
3000, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 3000,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 3000, false);
// load
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 3001, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -2523,7 +2574,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
3002);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
//4.config single item
config::delete_bitmap_lock_v2_white_list = "instance_id40";
@@ -2539,15 +2590,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 4000);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 4000,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 4000, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
4000);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
4000, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 4000,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 4000, false);
// load
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 4001, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -2561,7 +2611,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
4002);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
//instance_id41->v1
instance_id_x = "instance_id41";
@@ -2610,15 +2660,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 5000);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 5000,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 5000, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
5000);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
5000, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 5000,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 5000, false);
// load
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 5001, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -2632,7 +2681,7 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
5002);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
//instance_id51->v2
instance_id_x = "instance_id51";
@@ -2646,15 +2695,14 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, -1, 5100);
ASSERT_EQ(res_code, MetaServiceCode::OK);
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -1,
true);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 5100,
true);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 5100, true);
test_start_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
"job_id123",
instance_id_x);
res_code = update_delete_bitmap(meta_service.get(), table_id, 3, 5, -1,
5100);
ASSERT_EQ(res_code, MetaServiceCode::OK);
test_commit_compaction_job(table_id, 2, 3, 5, TabletCompactionJobPB::BASE,
5100, "job_id123",
instance_id_x);
- check_compaction_key(meta_service.get(), instance_id_x, table_id, 5100,
false);
- clear_rowsets(5);
+ check_job_key(meta_service.get(), instance_id_x, table_id, 5100, false);
// load
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 5101, -1);
ASSERT_EQ(res_code, MetaServiceCode::OK);
@@ -2668,10 +2716,11 @@ TEST(MetaServiceJobTest,
DeleteBitmapUpdateLockCompatibilityTest) {
check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
5102);
ASSERT_EQ(res_code, MetaServiceCode::OK);
- check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
false);
+ check_delete_bitmap_lock(meta_service.get(), instance_id_x, table_id, -2,
true);
}
-TEST(MetaServiceJobTest, CompactionJobWithMoWTest) {
+void testCompactionJobWithMoWTest(int lock_version) {
+ config::delete_bitmap_lock_v2_white_list = lock_version == 1 ? "" : "*";
auto meta_service = get_meta_service();
auto sp = SyncPoint::get_instance();
DORIS_CLOUD_DEFER {
@@ -2941,6 +2990,11 @@ TEST(MetaServiceJobTest, CompactionJobWithMoWTest) {
clear_rowsets(6);
}
+TEST(MetaServiceJobTest, CompactionJobWithMoWTest) {
+ testCompactionJobWithMoWTest(2);
+ testCompactionJobWithMoWTest(1);
+}
+
TEST(MetaServiceJobTest, SchemaChangeJobTest) {
auto meta_service = get_meta_service();
// meta_service->resource_mgr().reset(); // Do not use resource manager
@@ -3332,7 +3386,8 @@ TEST(MetaServiceJobTest, RetrySchemaChangeJobTest) {
}
}
-TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) {
+void testSchemaChangeJobWithMoWTest(int lock_version) {
+ config::delete_bitmap_lock_v2_white_list = lock_version == 1 ? "" : "*";
auto meta_service = get_meta_service();
// meta_service->resource_mgr().reset(); // Do not use resource manager
@@ -3433,7 +3488,8 @@ TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) {
output_rowsets, res);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
12345);
- ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
+ ASSERT_EQ(res_code,
+ lock_version == 1 ? MetaServiceCode::LOCK_EXPIRED :
MetaServiceCode::OK);
res.Clear();
// pending delete bitmap key on new tablet should be removed after
schema change job finishes
@@ -3465,7 +3521,8 @@ TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) {
output_rowsets, res);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
12345);
- ASSERT_EQ(res_code, MetaServiceCode::LOCK_EXPIRED);
+ ASSERT_EQ(res_code,
+ lock_version == 1 ? MetaServiceCode::LOCK_EXPIRED :
MetaServiceCode::OK);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, -2,
12346);
ASSERT_EQ(res_code, MetaServiceCode::OK);
res.Clear();
@@ -3495,6 +3552,11 @@ TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) {
}
}
+TEST(MetaServiceJobTest, SchemaChangeJobWithMoWTest) {
+ testSchemaChangeJobWithMoWTest(2);
+ testSchemaChangeJobWithMoWTest(1);
+}
+
TEST(MetaServiceJobTest, ConcurrentCompactionTest) {
auto meta_service = get_meta_service();
// meta_service->resource_mgr().reset(); // Do not use resource manager
diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp
index c7d1edc0ee1..e49b3c0deb9 100644
--- a/cloud/test/meta_service_test.cpp
+++ b/cloud/test/meta_service_test.cpp
@@ -4652,20 +4652,18 @@ TEST(MetaServiceTest, GetTabletStatsTest) {
}
void remove_delete_bitmap_lock(MetaServiceProxy* meta_service, int64_t
table_id) {
- std::string lock_key =
- meta_delete_bitmap_update_lock_key({"test_cloud_unique_id",
table_id, -1});
+ std::string lock_key =
meta_delete_bitmap_update_lock_key({"test_instance", table_id, -1});
std::unique_ptr<Transaction> txn;
ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
txn->remove(lock_key);
- std::string tablet_compaction_key_begin =
- mow_tablet_compaction_key({"test_cloud_unique_id", table_id, 0});
- std::string tablet_compaction_key_end =
- mow_tablet_compaction_key({"test_cloud_unique_id", table_id,
INT64_MAX});
- txn->remove(tablet_compaction_key_begin, tablet_compaction_key_end);
+ std::string tablet_job_key_begin = mow_tablet_job_key({"test_instance",
table_id, 0});
+ std::string tablet_job_key_end = mow_tablet_job_key({"test_instance",
table_id, INT64_MAX});
+ txn->remove(tablet_job_key_begin, tablet_job_key_end);
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
}
-TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
+void testGetDeleteBitmapUpdateLock(int lock_version, int job_lock_id) {
+ config::delete_bitmap_lock_v2_white_list = lock_version == 1 ? "" : "*";
auto meta_service = get_meta_service();
[[maybe_unused]] auto sp = SyncPoint::get_instance();
DORIS_CLOUD_DEFER {
@@ -4703,14 +4701,14 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
// case 2: lock key does not exist, get and remove compaction lock
req.add_partition_ids(123);
req.set_expiration(600);
- req.set_lock_id(-1);
+ req.set_lock_id(job_lock_id);
req.set_initiator(100);
meta_service->get_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
remove_req.set_tablet_id(2);
- remove_req.set_lock_id(-1);
+ remove_req.set_lock_id(job_lock_id);
remove_req.set_initiator(100);
meta_service->remove_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&remove_req, &remove_res,
@@ -4732,7 +4730,7 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_CONFLICT);
// case 4: lock key owned by load1, compaction1 get lock
- req.set_lock_id(-1);
+ req.set_lock_id(job_lock_id);
req.set_initiator(100);
meta_service->get_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
@@ -4778,14 +4776,14 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
sleep(2);
- req.set_lock_id(-1);
+ req.set_lock_id(job_lock_id);
req.set_initiator(888);
req.set_expiration(1);
meta_service->get_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- remove_req.set_lock_id(-1);
+ remove_req.set_lock_id(job_lock_id);
remove_req.set_initiator(888);
meta_service->remove_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&remove_req, &remove_res,
@@ -4793,14 +4791,14 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
ASSERT_EQ(remove_res.status().code(), MetaServiceCode::OK);
// case 7: lock key owned by compaction, new compaction get lock
- req.set_lock_id(-1);
+ req.set_lock_id(job_lock_id);
req.set_initiator(100);
req.set_expiration(100);
meta_service->get_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
- req.set_lock_id(-1);
+ req.set_lock_id(job_lock_id);
req.set_initiator(101);
req.set_expiration(1);
meta_service->get_delete_bitmap_update_lock(
@@ -4820,7 +4818,7 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::LOCK_CONFLICT);
- remove_req.set_lock_id(-1);
+ remove_req.set_lock_id(job_lock_id);
remove_req.set_initiator(100);
meta_service->remove_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&remove_req, &remove_res,
@@ -4829,7 +4827,7 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
// case 9: lock key owned by compaction but all expired (101 900), load1
get lock
req.set_table_id(table_id);
- req.set_lock_id(-1);
+ req.set_lock_id(job_lock_id);
req.set_initiator(900);
req.set_expiration(1);
meta_service->get_delete_bitmap_update_lock(
@@ -4852,28 +4850,91 @@ TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
nullptr);
ASSERT_EQ(remove_res.status().code(), MetaServiceCode::OK);
- // case 10: lock key does not exist, compaction get lock but txn commit
conflict, do fast retry
+ // case 10: lock key owned by compaction but all expired (101 900), schema
change get lock
+ req.set_table_id(table_id);
+ req.set_lock_id(job_lock_id);
+ req.set_initiator(900);
+ req.set_expiration(1);
+ meta_service->get_delete_bitmap_update_lock(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+ sleep(2);
+ req.set_table_id(table_id);
+ int other_job_lock_id = job_lock_id == COMPACTION_DELETE_BITMAP_LOCK_ID
+ ? SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID
+ : COMPACTION_DELETE_BITMAP_LOCK_ID;
+ req.set_lock_id(other_job_lock_id);
+ req.set_initiator(100);
+ req.set_expiration(1);
+ meta_service->get_delete_bitmap_update_lock(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+ remove_req.set_lock_id(other_job_lock_id);
+ remove_req.set_initiator(100);
+ meta_service->remove_delete_bitmap_update_lock(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&remove_req, &remove_res,
+ nullptr);
+ ASSERT_EQ(remove_res.status().code(), MetaServiceCode::OK);
+
+ // case 11: lock by schema change but expired, compaction get lock but txn
commit conflict, do fast retry
sp->set_call_back("get_delete_bitmap_update_lock:commit:conflict",
[&](auto&& args) {
auto* first_retry = try_any_cast<bool*>(args[0]);
- if (*first_retry) {
- *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_CONFLICT;
+ auto lock_id = (try_any_cast<const
GetDeleteBitmapUpdateLockRequest*>(args[1]))->lock_id();
+ if (*first_retry && is_job_delete_bitmap_lock_id(lock_id)) {
+ *try_any_cast<TxnErrorCode*>(args[2]) = TxnErrorCode::TXN_CONFLICT;
} else {
- *try_any_cast<TxnErrorCode*>(args[1]) = TxnErrorCode::TXN_OK;
+ *try_any_cast<TxnErrorCode*>(args[2]) = TxnErrorCode::TXN_OK;
}
});
sp->enable_processing();
- req.set_lock_id(-1);
+ req.set_lock_id(job_lock_id);
req.set_initiator(100);
req.set_expiration(10);
meta_service->get_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ remove_req.set_lock_id(job_lock_id);
+ remove_req.set_initiator(100);
+ meta_service->remove_delete_bitmap_update_lock(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&remove_req, &remove_res,
+ nullptr);
+ ASSERT_EQ(remove_res.status().code(), MetaServiceCode::OK);
+
+ // case 12: lock by load but expired, compaction get lock but txn commit
conflict, do fast retry
+ req.set_lock_id(300);
+ req.set_initiator(-1);
+ req.set_expiration(1);
+ meta_service->get_delete_bitmap_update_lock(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
+ sleep(2);
+ req.set_lock_id(job_lock_id);
+ req.set_initiator(100);
+ req.set_expiration(10);
+ meta_service->get_delete_bitmap_update_lock(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+ remove_delete_bitmap_lock(meta_service.get(), table_id);
+
+ // case 13: lock key does not exist, compaction get lock but txn commit
conflict, do fast retry
+ meta_service->get_delete_bitmap_update_lock(
+ reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req,
&res, nullptr);
+ ASSERT_EQ(res.status().code(), MetaServiceCode::OK);
+
sp->clear_all_call_backs();
sp->clear_trace();
sp->disable_processing();
+ remove_delete_bitmap_lock(meta_service.get(), table_id);
+}
- remove_delete_bitmap_lock(meta_service.get(), 1);
- remove_delete_bitmap_lock(meta_service.get(), 2);
+TEST(MetaServiceTest, GetDeleteBitmapUpdateLock) {
+ testGetDeleteBitmapUpdateLock(2, COMPACTION_DELETE_BITMAP_LOCK_ID);
+ testGetDeleteBitmapUpdateLock(2, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID);
+ testGetDeleteBitmapUpdateLock(1, COMPACTION_DELETE_BITMAP_LOCK_ID);
+ testGetDeleteBitmapUpdateLock(1, SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID);
}
TEST(MetaServiceTest, GetDeleteBitmapUpdateLockNoReadStats) {
@@ -5687,7 +5748,8 @@ TEST(MetaServiceTest,
UpdateDeleteBitmapScOverrideExistingKey) {
}
}
-TEST(MetaServiceTest, UpdateDeleteBitmap) {
+void testUpdateDeleteBitmap(int lock_version) {
+ config::delete_bitmap_lock_v2_white_list = lock_version == 1 ? "" : "*";
auto meta_service = get_meta_service();
remove_delete_bitmap_lock(meta_service.get(), 112);
@@ -5999,9 +6061,11 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) {
&remove_lock_res, nullptr);
ASSERT_EQ(remove_lock_res.status().code(), MetaServiceCode::OK);
- {
- // case: compaction update delete bitmap
- get_lock_req.set_lock_id(-1);
+ for (int i = 0; i < 2; ++i) {
+ auto lock_id =
+ i == 0 ? COMPACTION_DELETE_BITMAP_LOCK_ID :
SCHEMA_CHANGE_DELETE_BITMAP_LOCK_ID;
+ // case: compaction or schema_change update delete bitmap
+ get_lock_req.set_lock_id(lock_id);
get_lock_req.set_initiator(800);
meta_service->get_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&get_lock_req,
@@ -6013,7 +6077,7 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) {
update_delete_bitmap_req.set_cloud_unique_id("test_cloud_unique_id");
update_delete_bitmap_req.set_table_id(112);
update_delete_bitmap_req.set_partition_id(123);
- update_delete_bitmap_req.set_lock_id(-1);
+ update_delete_bitmap_req.set_lock_id(lock_id);
update_delete_bitmap_req.set_initiator(800);
update_delete_bitmap_req.set_tablet_id(333);
update_delete_bitmap_req.add_rowset_ids("123");
@@ -6025,15 +6089,15 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) {
&update_delete_bitmap_req, &update_delete_bitmap_res, nullptr);
ASSERT_EQ(update_delete_bitmap_res.status().code(),
MetaServiceCode::OK);
// remove lock
- remove_lock_req.set_lock_id(-1);
+ remove_lock_req.set_lock_id(lock_id);
remove_lock_req.set_initiator(800);
meta_service->remove_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&remove_lock_req,
&remove_lock_res, nullptr);
ASSERT_EQ(remove_lock_res.status().code(), MetaServiceCode::OK);
- // case: compaction update delete bitmap with lock expired
- get_lock_req.set_lock_id(-1);
+ // case: compaction or schema_change update delete bitmap with lock
expired
+ get_lock_req.set_lock_id(lock_id);
get_lock_req.set_initiator(800);
get_lock_req.set_expiration(1);
meta_service->get_delete_bitmap_update_lock(
@@ -6055,9 +6119,9 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) {
&update_delete_bitmap_req, &update_delete_bitmap_res, nullptr);
ASSERT_EQ(update_delete_bitmap_res.status().code(),
MetaServiceCode::LOCK_EXPIRED);
- // case: compaction2 get lock
+ // case: compaction2 or schema_change2 get lock
sleep(2);
- get_lock_req.set_lock_id(-1);
+ get_lock_req.set_lock_id(lock_id);
get_lock_req.set_initiator(810);
get_lock_req.set_expiration(1);
meta_service->get_delete_bitmap_update_lock(
@@ -6069,8 +6133,8 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) {
reinterpret_cast<google::protobuf::RpcController*>(&cntl),
&update_delete_bitmap_req, &update_delete_bitmap_res, nullptr);
ASSERT_EQ(update_delete_bitmap_res.status().code(),
MetaServiceCode::LOCK_EXPIRED);
- // remove compaction2 lock
- remove_lock_req.set_lock_id(-1);
+ // remove compaction2 or or schema_change2 lock
+ remove_lock_req.set_lock_id(lock_id);
remove_lock_req.set_initiator(810);
meta_service->remove_delete_bitmap_update_lock(
reinterpret_cast<::google::protobuf::RpcController*>(&cntl),
&remove_lock_req,
@@ -6125,6 +6189,11 @@ TEST(MetaServiceTest, UpdateDeleteBitmap) {
remove_delete_bitmap_lock(meta_service.get(), 112);
}
+TEST(MetaServiceTest, UpdateDeleteBitmap) {
+ testUpdateDeleteBitmap(2);
+ testUpdateDeleteBitmap(1);
+}
+
TEST(MetaServiceTest, UpdateDeleteBitmapWithException) {
auto meta_service = get_meta_service();
brpc::Controller cntl;
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index ff1b33bf9f1..e720cebad43 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -463,13 +463,12 @@ static int create_delete_bitmap_update_lock_kv(TxnKv*
txn_kv, int64_t table_id,
return -1;
}
txn->put(key, val);
- std::string tablet_compaction_key =
- mow_tablet_compaction_key({instance_id, table_id, initiator});
- std::string tablet_compaction_val;
- MowTabletCompactionPB mow_tablet_compaction;
- mow_tablet_compaction.set_expiration(expiration);
- mow_tablet_compaction.SerializeToString(&tablet_compaction_val);
- txn->put(tablet_compaction_key, tablet_compaction_val);
+ std::string tablet_job_key = mow_tablet_job_key({instance_id, table_id,
initiator});
+ std::string tablet_job_val;
+ MowTabletJobPB mow_tablet_job;
+ mow_tablet_job.set_expiration(expiration);
+ mow_tablet_job.SerializeToString(&tablet_job_val);
+ txn->put(tablet_job_key, tablet_job_val);
if (txn->commit() != TxnErrorCode::TXN_OK) {
return -1;
}
@@ -1439,10 +1438,9 @@ TEST(RecyclerTest, recycle_versions) {
std::string delete_bitmap_update_lock_val;
ASSERT_EQ(txn->get(delete_bitmap_update_lock_key,
&delete_bitmap_update_lock_val),
TxnErrorCode::TXN_OK);
- auto tablet_compaction_key0 = mow_tablet_compaction_key({instance_id,
table_id, 0});
- auto tablet_compaction_key1 = mow_tablet_compaction_key({instance_id,
table_id + 1, 0});
- ASSERT_EQ(txn->get(tablet_compaction_key0, tablet_compaction_key1, &iter),
- TxnErrorCode::TXN_OK);
+ auto tablet_job_key0 = mow_tablet_job_key({instance_id, table_id, 0});
+ auto tablet_job_key1 = mow_tablet_job_key({instance_id, table_id + 1, 0});
+ ASSERT_EQ(txn->get(tablet_job_key0, tablet_job_key1, &iter),
TxnErrorCode::TXN_OK);
ASSERT_EQ(iter->size(), 2);
// Drop indexes
@@ -1461,8 +1459,7 @@ TEST(RecyclerTest, recycle_versions) {
// delete bitmap update lock must be deleted
ASSERT_EQ(txn->get(delete_bitmap_update_lock_key,
&delete_bitmap_update_lock_val),
TxnErrorCode::TXN_KEY_NOT_FOUND);
- ASSERT_EQ(txn->get(tablet_compaction_key0, tablet_compaction_key1, &iter),
- TxnErrorCode::TXN_OK);
+ ASSERT_EQ(txn->get(tablet_job_key0, tablet_job_key1, &iter),
TxnErrorCode::TXN_OK);
ASSERT_EQ(iter->size(), 0);
}
@@ -3358,19 +3355,18 @@ MetaServiceCode
remove_delete_bitmap_lock(MetaServiceProxy* meta_service, int64_
}
void remove_delete_bitmap_lock(MetaServiceProxy* meta_service, int64_t
table_id) {
- std::string lock_key =
- meta_delete_bitmap_update_lock_key({"test_check_compaction_key",
table_id, -1});
+ std::string lock_key =
meta_delete_bitmap_update_lock_key({"test_check_job_key", table_id, -1});
std::unique_ptr<Transaction> txn;
ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK);
txn->remove(lock_key);
ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
-}
+} // namespace
-TEST(CheckerTest, check_compaction_key) {
- config::enable_mow_compaction_key_check = true;
- config::compaction_key_check_expiration_diff_seconds = 0;
+TEST(CheckerTest, check_job_key) {
+ config::enable_mow_job_key_check = true;
+ config::mow_job_key_check_expiration_diff_seconds = 0;
config::delete_bitmap_lock_v2_white_list = "*";
- std::string instance_id = "test_check_compaction_key";
+ std::string instance_id = "test_check_job_key";
[[maybe_unused]] auto sp = SyncPoint::get_instance();
DORIS_CLOUD_DEFER {
SyncPoint::get_instance()->clear_all_call_backs();
@@ -3402,7 +3398,7 @@ TEST(CheckerTest, check_compaction_key) {
obj_info->set_id("1");
InstanceChecker checker(meta_service->txn_kv(), instance_id);
ASSERT_EQ(checker.init(instance), 0);
- ASSERT_EQ(checker.do_mow_compaction_key_check(), 0);
+ ASSERT_EQ(checker.do_mow_job_key_check(), 0);
res_code = remove_delete_bitmap_lock(meta_service.get(), table_id, 100,
-1);
//test 2:
@@ -3416,7 +3412,7 @@ TEST(CheckerTest, check_compaction_key) {
std::this_thread::sleep_for(std::chrono::seconds(1));
remove_delete_bitmap_lock(meta_service.get(), table_id);
res_code = get_delete_bitmap_lock(meta_service.get(), table_id, 101, -1);
- ASSERT_EQ(checker.do_mow_compaction_key_check(), -1);
+ ASSERT_EQ(checker.do_mow_job_key_check(), -1);
std::this_thread::sleep_for(std::chrono::seconds(6));
//test 3:
@@ -3429,7 +3425,7 @@ TEST(CheckerTest, check_compaction_key) {
ASSERT_EQ(res_code, MetaServiceCode::OK);
std::this_thread::sleep_for(std::chrono::seconds(1));
remove_delete_bitmap_lock(meta_service.get(), table_id);
- ASSERT_EQ(checker.do_mow_compaction_key_check(), -1);
+ ASSERT_EQ(checker.do_mow_job_key_check(), -1);
}
TEST(CheckerTest, delete_bitmap_storage_optimize_v2_check_normal) {
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index 4180c8e8e9a..b4ed46679f1 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -1515,7 +1515,7 @@ message DeleteBitmapUpdateLockPB {
repeated int64 initiators = 3;
}
-message MowTabletCompactionPB {
+message MowTabletJobPB {
optional int64 expiration = 1;
}
diff --git
a/regression-test/suites/schema_change_p0/test_unique_schema_key_change_multi_tablets.groovy
b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_multi_tablets.groovy
new file mode 100644
index 00000000000..fb88ae6a739
--- /dev/null
+++
b/regression-test/suites/schema_change_p0/test_unique_schema_key_change_multi_tablets.groovy
@@ -0,0 +1,391 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.apache.doris.regression.suite.ClusterOptions
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+suite("test_unique_schema_key_change_multi_tablets","docker") {
+ def options = new ClusterOptions()
+ options.cloudMode = true
+ options.setFeNum(1)
+ options.setBeNum(1)
+ options.enableDebugPoints()
+ options.feConfigs.add("enable_workload_group=false")
+ options.beConfigs.add('enable_java_support=false')
+ options.beConfigs.add('meta_service_conflict_error_retry_times=1')
+ options.beConfigs.add('alter_tablet_worker_count=100')
+ options.msConfigs.add('delete_bitmap_lock_v2_white_list=*')
+ options.msConfigs.add('enable_retry_txn_conflict=false')
+
+ def tbName = "test_unique_schema_key_change_multi_tablets"
+ sql """ DROP TABLE IF EXISTS ${tbName} """
+ def initTable = " CREATE TABLE `${tbName}` (\n" +
+ " `source` int NOT NULL,\n" +
+ " `data_region` varchar(6) NULL,\n" +
+ " `report_date` date NOT NULL,\n" +
+ " `report_hour` varchar(20),\n" +
+ " `affiliate_id` varchar(200),\n" +
+ " `ad_format` varchar(200) NULL,\n" +
+ " `ad_width` varchar(200) NULL,\n" +
+ " `ad_height` varchar(200) NULL,\n" +
+ " `os` varchar(200) NULL,\n" +
+ " `device_make` varchar(200) NULL,\n" +
+ " `bundle_id` varchar(200) NULL,\n" +
+ " `country` varchar(200) NULL,\n" +
+ " `country_a2` varchar(200) NULL,\n" +
+ " `connection_type` varchar(200) NULL,\n" +
+ " `campaign_id` int NULL,\n" +
+ " `ad_group_id` int NULL,\n" +
+ " `ad_id` int NULL,\n" +
+ " `creative_id` int NULL,\n" +
+ " `adv_id` int NULL,\n" +
+ " `offer_id` varchar(200) NULL,\n" +
+ " `bop_aff_id` varchar(200) NULL,\n" +
+ " `bid_strategy` int NULL,\n" +
+ " `p_ctr_version` varchar(200) NULL,\n" +
+ " `p_cvr_version` varchar(200) NULL,\n" +
+ " `p_d_cvr_version` varchar(200) NULL,\n" +
+ " `p_open_ivr_version` varchar(200) NULL,\n" +
+ " `feature_1` varchar(200) NULL,\n" +
+ " `response_type` int NULL,\n" +
+ " `pos` int NULL,\n" +
+ " `instl` int NULL,\n" +
+ " `domain` varchar(200) NULL,\n" +
+ " `video_placement` int NULL,\n" +
+ " `is_rewarded` int NULL,\n" +
+ " `bid_algorithm` int NULL,\n" +
+ " `first_ssp` varchar(200) NULL,\n" +
+ " `tag_id` varchar(512) NULL,\n" +
+ " `publisher_id` varchar(200) NULL,\n" +
+ " `fraud_type` varchar(200) NULL,\n" +
+ " `empty_device_id` int NULL,\n" +
+ " `audience` varchar(200) NULL,\n" +
+ " `opt_price` varchar(200) NULL,\n" +
+ " `package` varchar(512) NULL,\n" +
+ " `cold_boot_type` varchar(64) NULL,\n" +
+ " `bid_mode` varchar(64) NULL,\n" +
+ " `not_boot_cause` varchar(64) NULL,\n" +
+ " `traffic_source` int NULL,\n" +
+ " `traffic_type` int NULL,\n" +
+ " `bid_not_adjust_cause` int NULL,\n" +
+ " `win_rate_target_roi` varchar(200) NULL,\n" +
+ " `cpi_reduction_flag` int NULL,\n" +
+ " `companion_ad` int NULL,\n" +
+ " `companion_ad_w` int NULL,\n" +
+ " `companion_ad_h` int NULL,\n" +
+ " `parent_ad_creative_id` int NULL,\n" +
+ " `adjust_bid_version` varchar(200) NULL,\n" +
+ " `win_rate_version` varchar(200) NULL,\n" +
+ " `cpi_ad_cold_sort_flag` int NULL,\n" +
+ " `cpi_ad_cold_sort_rule` int NULL,\n" +
+ " `cps_cool_start_mode` varchar(32) NULL,\n" +
+ " `deeplink_support` int NULL,\n" +
+ " `ad_group_parent_id` varchar(200) NULL,\n" +
+ " `ad_parent_id` varchar(200) NULL,\n" +
+ " `category1_id` int NULL,\n" +
+ " `category1_name` varchar(200) NULL,\n" +
+ " `category2_id` int NULL,\n" +
+ " `category2_name` varchar(200) NULL,\n" +
+ " `category3_id` int NULL,\n" +
+ " `category3_name` varchar(200) NULL,\n" +
+ " `package_category1_id` int NULL,\n" +
+ " `package_category1_name` varchar(200) NULL,\n" +
+ " `package_category2_id` int NULL,\n" +
+ " `package_category2_name` varchar(200) NULL,\n" +
+ " `package_category3_id` int NULL,\n" +
+ " `package_category3_name` varchar(200) NULL,\n" +
+ " `affiliate_name` varchar(200) NULL,\n" +
+ " `campaign_name` varchar(200) NULL,\n" +
+ " `ad_group_name` varchar(200) NULL,\n" +
+ " `ad_name` varchar(200) NULL,\n" +
+ " `creative_name` varchar(200) NULL,\n" +
+ " `adv_name` varchar(200) NULL,\n" +
+ " `adv_type` varchar(200) NULL,\n" +
+ " `is_oem` int NULL,\n" +
+ " `tag_name` varchar(200) NULL,\n" +
+ " `request_count` bigint NULL,\n" +
+ " `response_count` bigint NULL,\n" +
+ " `rta_request` int NULL,\n" +
+ " `rta_request_true` int NULL,\n" +
+ " `shein_rta_request` int NULL,\n" +
+ " `shein_rta_request_true` int NULL,\n" +
+ " `tiktok_rta_request` int NULL,\n" +
+ " `tiktok_rta_request_true` int NULL,\n" +
+ " `win_count` bigint NULL,\n" +
+ " `imp_count` bigint NULL,\n" +
+ " `companion_imp_count` bigint NULL,\n" +
+ " `original_imp_count` bigint NULL,\n" +
+ " `click_count` bigint NULL,\n" +
+ " `companion_click_count` bigint NULL,\n" +
+ " `original_click_count` bigint NULL,\n" +
+ " `bid_floor_total` decimal(20,5) NULL,\n" +
+ " `bid_price_total` decimal(20,5) NULL,\n" +
+ " `bid_price_exp_total` decimal(20,5) NULL,\n" +
+ " `p_ctr_total` decimal(20,5) NULL,\n" +
+ " `p_ctr_exp_total` decimal(20,5) NULL,\n" +
+ " `p_ctr_calibrate_exp_total` decimal(20,5) NULL,\n" +
+ " `p_cvr_total` decimal(20,5) NULL,\n" +
+ " `p_cvr_imp_total` decimal(20,5) NULL,\n" +
+ " `p_cvr_ck_total` decimal(20,5) NULL,\n" +
+ " `p_d_cvr_total` decimal(20,5) NULL,\n" +
+ " `p_d_cvr_ck_total` decimal(20,5) NULL,\n" +
+ " `p_open_ivr_res_total` decimal(20,5) NULL,\n" +
+ " `p_open_ivr_imp_total` decimal(20,5) NULL,\n" +
+ " `p_open_ivr_ck_total` decimal(20,5) NULL,\n" +
+ " `win_success_price_total` decimal(20,5) NULL,\n" +
+ " `imp_success_price_total` decimal(20,5) NULL,\n" +
+ " `price_total` decimal(20,5) NULL,\n" +
+ " `event1_count` bigint NULL,\n" +
+ " `event2_count` bigint NULL,\n" +
+ " `event3_count` bigint NULL,\n" +
+ " `event4_count` bigint NULL,\n" +
+ " `event5_count` bigint NULL,\n" +
+ " `event6_count` bigint NULL,\n" +
+ " `event7_count` bigint NULL,\n" +
+ " `event8_count` bigint NULL,\n" +
+ " `event9_count` bigint NULL,\n" +
+ " `event10_count` bigint NULL,\n" +
+ " `event10_count_day0` bigint NULL,\n" +
+ " `event11_count` bigint NULL,\n" +
+ " `ad_estimated_commission` decimal(20,5) NULL,\n" +
+ " `adv_opt_price` decimal(20,5) NULL,\n" +
+ " `event10_count_rt` bigint NULL,\n" +
+ " `event11_count_rt` bigint NULL,\n" +
+ " `ad_estimated_commission_rt` decimal(20,5) NULL,\n" +
+ " `adv_opt_price_rt` decimal(20,5) NULL,\n" +
+ " `event11_day0` bigint NULL,\n" +
+ " `event11_day1` bigint NULL,\n" +
+ " `event11_day2` bigint NULL,\n" +
+ " `event11_day3` bigint NULL,\n" +
+ " `ad_estimated_commission_day0` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_day1` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_day2` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_day3` decimal(20,5) NULL,\n" +
+ " `request_error_count` bigint NULL,\n" +
+ " `request_filter_count` bigint NULL,\n" +
+ " `p_reward` decimal(20,5) NULL,\n" +
+ " `request_id_count` bigint NULL,\n" +
+ " `imp_id_count` bigint NULL,\n" +
+ " `register_cnt` bigint NULL,\n" +
+ " `pay_cnt` bigint NULL,\n" +
+ " `2days_login_cnt` bigint NULL,\n" +
+ " `key_action_cnt` bigint NULL,\n" +
+ " `key_action2_cnt` bigint NULL,\n" +
+ " `key_action3_cnt` bigint NULL,\n" +
+ " `login_app_cnt` bigint NULL,\n" +
+ " `p_ct_cvr_imp_total` decimal(20,5) NULL,\n" +
+ " `p_ct_cvr_respond_total` decimal(20,5) NULL,\n" +
+ " `p_ct_cvr_click_total` decimal(20,5) NULL,\n" +
+ " `p_d_cvr_event10_total` decimal(20,5) NULL,\n" +
+ " `install_cnt` bigint NULL,\n" +
+ " `ecpm_total` decimal(20,5) NULL,\n" +
+ " `ecpm_before_adjust_total` decimal(20,5) NULL,\n" +
+ " `ecpm_after_adjust_total` decimal(20,5) NULL,\n" +
+ " `ecpm_before_adjust_imp_total` decimal(20,5) NULL,\n" +
+ " `p_rate_opt_total` decimal(20,5) NULL,\n" +
+ " `p_rate_imp_opt_total` decimal(20,5) NULL,\n" +
+ " `win_rate_lambda_total` decimal(20,5) NULL,\n" +
+ " `win_rate_mu_total` decimal(20,5) NULL,\n" +
+ " `win_rate_sigma_total` decimal(20,5) NULL,\n" +
+ " `imp_fj_ck` bigint NULL,\n" +
+ " `imp_fj_cfm` bigint NULL,\n" +
+ " `launch_success` bigint NULL,\n" +
+ " `launch_failed` bigint NULL,\n" +
+ " `purchase_event_a_cnt` bigint NULL,\n" +
+ " `purchase_event_c_cnt` bigint NULL,\n" +
+ " `purchase_event_a_cnt_art` bigint NULL,\n" +
+ " `purchase_event_c_cnt_art` bigint NULL,\n" +
+ " `app_open_cnt_art` bigint NULL,\n" +
+ " `order_amount` decimal(20,5) NULL,\n" +
+ " `order_amount_art` decimal(20,5) NULL,\n" +
+ " `adv_spend` decimal(20,5) NULL,\n" +
+ " `adv_spend_art` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_art` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_art_day0` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_art_day1` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_art_day2` decimal(20,5) NULL,\n" +
+ " `ad_estimated_commission_art_day3` decimal(20,5) NULL,\n" +
+ " `imp_count_et` bigint NULL,\n" +
+ " `click_count_et` bigint NULL,\n" +
+ " `purchase_event_a_cnt_et` bigint NULL,\n" +
+ " `purchase_event_c_cnt_et` bigint NULL,\n" +
+ " `purchase_event_a_cnt_art_et` bigint NULL,\n" +
+ " `purchase_event_c_cnt_art_et` bigint NULL,\n" +
+ " `app_open_cnt_art_et` bigint NULL,\n" +
+ " `order_amount_et` decimal(20,5) NULL,\n" +
+ " `order_amount_art_et` decimal(20,5) NULL,\n" +
+ " `adv_spend_et` decimal(20,5) NULL,\n" +
+ " `adv_spend_art_et` decimal(20,5) NULL,\n" +
+ " `event11_count_pt` bigint NULL,\n" +
+ " `ad_estimated_commission_pt` decimal(20,5) NULL,\n" +
+ " `etl_time` datetime NULL COMMENT 'ETL时间'\n" +
+ ") ENGINE=OLAP\n" +
+ "UNIQUE KEY(`source`, `data_region`, `report_date`,
`report_hour`, `affiliate_id`, `ad_format`, `ad_width`, `ad_height`, `os`,
`device_make`, `bundle_id`, `country`, `country_a2`, `connection_type`,
`campaign_id`, `ad_group_id`, `ad_id`, `creative_id`, `adv_id`, `offer_id`,
`bop_aff_id`, `bid_strategy`, `p_ctr_version`, `p_cvr_version`,
`p_d_cvr_version`, `p_open_ivr_version`, `feature_1`, `response_type`, `pos`,
`instl`, `domain`, `video_placement`, `is_rewarded`, `bid_algo [...]
+ "PARTITION BY RANGE(`report_date`)\n" +
+ "(PARTITION p20250101 VALUES [('2025-01-01'), ('2025-01-02')),\n"
+
+ "PARTITION p20250102 VALUES [('2025-01-02'), ('2025-01-03')),\n" +
+ "PARTITION p20250103 VALUES [('2025-01-03'), ('2025-01-04')),\n" +
+ "PARTITION p20250104 VALUES [('2025-01-04'), ('2025-01-05')),\n" +
+ "PARTITION p20250105 VALUES [('2025-01-05'), ('2025-01-06')),\n" +
+ "PARTITION p20250106 VALUES [('2025-01-06'), ('2025-01-07')),\n" +
+ "PARTITION p20250107 VALUES [('2025-01-07'), ('2025-01-08')),\n" +
+ "PARTITION p20250108 VALUES [('2025-01-08'), ('2025-01-09')),\n" +
+ "PARTITION p20250109 VALUES [('2025-01-09'), ('2025-01-10')),\n" +
+ "PARTITION p20250110 VALUES [('2025-01-10'), ('2025-01-11')),\n" +
+ "PARTITION p20250111 VALUES [('2025-01-11'), ('2025-01-12')),\n" +
+ "PARTITION p20250112 VALUES [('2025-01-12'), ('2025-01-13')),\n" +
+ "PARTITION p20250113 VALUES [('2025-01-13'), ('2025-01-14')),\n" +
+ "PARTITION p20250114 VALUES [('2025-01-14'), ('2025-01-15')),\n" +
+ "PARTITION p20250115 VALUES [('2025-01-15'), ('2025-01-16')),\n" +
+ "PARTITION p20250116 VALUES [('2025-01-16'), ('2025-01-17')),\n" +
+ "PARTITION p20250117 VALUES [('2025-01-17'), ('2025-01-18')),\n" +
+ "PARTITION p20250118 VALUES [('2025-01-18'), ('2025-01-19')),\n" +
+ "PARTITION p20250119 VALUES [('2025-01-19'), ('2025-01-20')),\n" +
+ "PARTITION p20250120 VALUES [('2025-01-20'), ('2025-01-21')),\n" +
+ "PARTITION p20250121 VALUES [('2025-01-21'), ('2025-01-22')),\n" +
+ "PARTITION p20250122 VALUES [('2025-01-22'), ('2025-01-23')),\n" +
+ "PARTITION p20250123 VALUES [('2025-01-23'), ('2025-01-24')),\n" +
+ "PARTITION p20250124 VALUES [('2025-01-24'), ('2025-01-25')),\n" +
+ "PARTITION p20250125 VALUES [('2025-01-25'), ('2025-01-26')),\n" +
+ "PARTITION p20250126 VALUES [('2025-01-26'), ('2025-01-27')),\n" +
+ "PARTITION p20250127 VALUES [('2025-01-27'), ('2025-01-28')),\n" +
+ "PARTITION p20250128 VALUES [('2025-01-28'), ('2025-01-29')),\n" +
+ "PARTITION p20250129 VALUES [('2025-01-29'), ('2025-01-30')),\n" +
+ "PARTITION p20250130 VALUES [('2025-01-30'), ('2025-01-31')),\n" +
+ "PARTITION p20250131 VALUES [('2025-01-31'), ('2025-02-01')),\n" +
+ "PARTITION p20250201 VALUES [('2025-02-01'), ('2025-02-02')),\n" +
+ "PARTITION p20250202 VALUES [('2025-02-02'), ('2025-02-03')),\n" +
+ "PARTITION p20250203 VALUES [('2025-02-03'), ('2025-02-04')),\n" +
+ "PARTITION p20250204 VALUES [('2025-02-04'), ('2025-02-05')),\n" +
+ "PARTITION p20250205 VALUES [('2025-02-05'), ('2025-02-06')),\n" +
+ "PARTITION p20250206 VALUES [('2025-02-06'), ('2025-02-07')),\n" +
+ "PARTITION p20250207 VALUES [('2025-02-07'), ('2025-02-08')),\n" +
+ "PARTITION p20250208 VALUES [('2025-02-08'), ('2025-02-09')),\n" +
+ "PARTITION p20250209 VALUES [('2025-02-09'), ('2025-02-10')),\n" +
+ "PARTITION p20250210 VALUES [('2025-02-10'), ('2025-02-11')),\n" +
+ "PARTITION p20250211 VALUES [('2025-02-11'), ('2025-02-12')),\n" +
+ "PARTITION p20250212 VALUES [('2025-02-12'), ('2025-02-13')),\n" +
+ "PARTITION p20250213 VALUES [('2025-02-13'), ('2025-02-14')),\n" +
+ "PARTITION p20250214 VALUES [('2025-02-14'), ('2025-02-15')),\n" +
+ "PARTITION p20250215 VALUES [('2025-02-15'), ('2025-02-16')),\n" +
+ "PARTITION p20250216 VALUES [('2025-02-16'), ('2025-02-17')),\n" +
+ "PARTITION p20250217 VALUES [('2025-02-17'), ('2025-02-18')),\n" +
+ "PARTITION p20250218 VALUES [('2025-02-18'), ('2025-02-19')),\n" +
+ "PARTITION p20250219 VALUES [('2025-02-19'), ('2025-02-20')),\n" +
+ "PARTITION p20250220 VALUES [('2025-02-20'), ('2025-02-21')),\n" +
+ "PARTITION p20250221 VALUES [('2025-02-21'), ('2025-02-22')),\n" +
+ "PARTITION p20250222 VALUES [('2025-02-22'), ('2025-02-23')),\n" +
+ "PARTITION p20250223 VALUES [('2025-02-23'), ('2025-02-24')),\n" +
+ "PARTITION p20250224 VALUES [('2025-02-24'), ('2025-02-25')),\n" +
+ "PARTITION p20250225 VALUES [('2025-02-25'), ('2025-02-26')),\n" +
+ "PARTITION p20250226 VALUES [('2025-02-26'), ('2025-02-27')),\n" +
+ "PARTITION p20250227 VALUES [('2025-02-27'), ('2025-02-28')),\n" +
+ "PARTITION p20250228 VALUES [('2025-02-28'), ('2025-03-01')),\n" +
+ "PARTITION p20250301 VALUES [('2025-03-01'), ('2025-03-02')),\n" +
+ "PARTITION p20250302 VALUES [('2025-03-02'), ('2025-03-03')),\n" +
+ "PARTITION p20250303 VALUES [('2025-03-03'), ('2025-03-04')),\n" +
+ "PARTITION p20250304 VALUES [('2025-03-04'), ('2025-03-05')),\n" +
+ "PARTITION p20250305 VALUES [('2025-03-05'), ('2025-03-06')),\n" +
+ "PARTITION p20250306 VALUES [('2025-03-06'), ('2025-03-07')),\n" +
+ "PARTITION p20250307 VALUES [('2025-03-07'), ('2025-03-08')),\n" +
+ "PARTITION p20250308 VALUES [('2025-03-08'), ('2025-03-09')),\n" +
+ "PARTITION p20250309 VALUES [('2025-03-09'), ('2025-03-10')),\n" +
+ "PARTITION p20250310 VALUES [('2025-03-10'), ('2025-03-11')),\n" +
+ "PARTITION p20250311 VALUES [('2025-03-11'), ('2025-03-12')),\n" +
+ "PARTITION p20250312 VALUES [('2025-03-12'), ('2025-03-13')),\n" +
+ "PARTITION p20250313 VALUES [('2025-03-13'), ('2025-03-14')),\n" +
+ "PARTITION p20250314 VALUES [('2025-03-14'), ('2025-03-15')),\n" +
+ "PARTITION p20250315 VALUES [('2025-03-15'), ('2025-03-16')),\n" +
+ "PARTITION p20250316 VALUES [('2025-03-16'), ('2025-03-17')),\n" +
+ "PARTITION p20250317 VALUES [('2025-03-17'), ('2025-03-18')),\n" +
+ "PARTITION p20250318 VALUES [('2025-03-18'), ('2025-03-19')),\n" +
+ "PARTITION p20250319 VALUES [('2025-03-19'), ('2025-03-20')),\n" +
+ "PARTITION p20250320 VALUES [('2025-03-20'), ('2025-03-21')),\n" +
+ "PARTITION p20250321 VALUES [('2025-03-21'), ('2025-03-22')),\n" +
+ "PARTITION p20250322 VALUES [('2025-03-22'), ('2025-03-23')),\n" +
+ "PARTITION p20250323 VALUES [('2025-03-23'), ('2025-03-24')),\n" +
+ "PARTITION p20250324 VALUES [('2025-03-24'), ('2025-03-25')),\n" +
+ "PARTITION p20250325 VALUES [('2025-03-25'), ('2025-03-26')),\n" +
+ "PARTITION p20250326 VALUES [('2025-03-26'), ('2025-03-27')),\n" +
+ "PARTITION p20250327 VALUES [('2025-03-27'), ('2025-03-28')),\n" +
+ "PARTITION p20250328 VALUES [('2025-03-28'), ('2025-03-29')),\n" +
+ "PARTITION p20250329 VALUES [('2025-03-29'), ('2025-03-30')),\n" +
+ "PARTITION p20250330 VALUES [('2025-03-30'), ('2025-03-31')),\n" +
+ "PARTITION p20250331 VALUES [('2025-03-31'), ('2025-04-01')),\n" +
+ "PARTITION p20250401 VALUES [('2025-04-01'), ('2025-04-02')),\n" +
+ "PARTITION p20250402 VALUES [('2025-04-02'), ('2025-04-03')),\n" +
+ "PARTITION p20250403 VALUES [('2025-04-03'), ('2025-04-04')),\n" +
+ "PARTITION p20250404 VALUES [('2025-04-04'), ('2025-04-05')),\n" +
+ "PARTITION p20250405 VALUES [('2025-04-05'), ('2025-04-06')),\n" +
+ "PARTITION p20250406 VALUES [('2025-04-06'), ('2025-04-07')),\n" +
+ "PARTITION p20250407 VALUES [('2025-04-07'), ('2025-04-08')),\n" +
+ "PARTITION p20250408 VALUES [('2025-04-08'), ('2025-04-09')),\n" +
+ "PARTITION p20250409 VALUES [('2025-04-09'), ('2025-04-10')),\n" +
+ "PARTITION p20250410 VALUES [('2025-04-10'), ('2025-04-11')),\n" +
+ "PARTITION p20250411 VALUES [('2025-04-11'), ('2025-04-12')),\n" +
+ "PARTITION p20250412 VALUES [('2025-04-12'), ('2025-04-13')),\n" +
+ "PARTITION p20250413 VALUES [('2025-04-13'), ('2025-04-14')),\n" +
+ "PARTITION p20250414 VALUES [('2025-04-14'), ('2025-04-15')),\n" +
+ "PARTITION p20250415 VALUES [('2025-04-15'), ('2025-04-16')),\n" +
+ "PARTITION p20250416 VALUES [('2025-04-16'), ('2025-04-17')),\n" +
+ "PARTITION p20250417 VALUES [('2025-04-17'), ('2025-04-18')),\n" +
+ "PARTITION p20250418 VALUES [('2025-04-18'), ('2025-04-19')),\n" +
+ "PARTITION p20250419 VALUES [('2025-04-19'), ('2025-04-20')),\n" +
+ "PARTITION p20250420 VALUES [('2025-04-20'), ('2025-04-21')),\n" +
+ "PARTITION p20250421 VALUES [('2025-04-21'), ('2025-04-22')),\n" +
+ "PARTITION p20250422 VALUES [('2025-04-22'), ('2025-04-23')),\n" +
+ "PARTITION p20250423 VALUES [('2025-04-23'), ('2025-04-24')),\n" +
+ "PARTITION p20250424 VALUES [('2025-04-24'), ('2025-04-25')),\n" +
+ "PARTITION p20250425 VALUES [('2025-04-25'), ('2025-04-26')),\n" +
+ "PARTITION p20250426 VALUES [('2025-04-26'), ('2025-04-27')),\n" +
+ "PARTITION p20250427 VALUES [('2025-04-27'), ('2025-04-28')),\n" +
+ "PARTITION p20250428 VALUES [('2025-04-28'), ('2025-04-29')),\n" +
+ "PARTITION p20250429 VALUES [('2025-04-29'), ('2025-04-30')),\n" +
+ "PARTITION p20250430 VALUES [('2025-04-30'), ('2025-05-01')),\n" +
+ "PARTITION p20250501 VALUES [('2025-05-01'), ('2025-05-02')),\n" +
+ "PARTITION p20250502 VALUES [('2025-05-02'), ('2025-05-03')),\n" +
+ "PARTITION p20250503 VALUES [('2025-05-03'), ('2025-05-04')),\n" +
+ "PARTITION p20250504 VALUES [('2025-05-04'), ('2025-05-05')),\n" +
+ "PARTITION p20250505 VALUES [('2025-05-05'), ('2025-05-06')),\n" +
+ "PARTITION p20250506 VALUES [('2025-05-06'), ('2025-05-07')),\n" +
+ "PARTITION p20250507 VALUES [('2025-05-07'), ('2025-05-08')),\n" +
+ "PARTITION p20250508 VALUES [('2025-05-08'), ('2025-05-09')),\n" +
+ "PARTITION p20250509 VALUES [('2025-05-09'), ('2025-05-10')),\n" +
+ "PARTITION p20250510 VALUES [('2025-05-10'), ('2025-05-11')),\n" +
+ "PARTITION p20250511 VALUES [('2025-05-11'), ('2025-05-12')),\n" +
+ "PARTITION p20250512 VALUES [('2025-05-12'), ('2025-05-13')))\n" +
+ "DISTRIBUTED BY HASH(`source`, `affiliate_id`, `ad_format`,
`ad_width`, `ad_height`, `bundle_id`, `country`, `campaign_id`, `ad_group_id`)
BUCKETS 24\n" +
+ "PROPERTIES (\"enable_unique_key_merge_on_write\" = \"true\");"
+
+ docker(options) {
+ sql initTable
+ sql """ alter table ${tbName} add column p_order_ivr_version
varchar(200) KEY NULL after p_open_ivr_version """
+
+ def result
+ for (int i = 0; i < 100; i++) {
+ result = sql_return_maparray """ SHOW ALTER TABLE COLUMN WHERE
TableName='${tbName}' ORDER BY createtime DESC LIMIT 1 """
+ logger.info("result: ${result}")
+ if (result.size() > 0 && (result[0].State == "FINISHED" ||
result[0].State == "CANCELLED")) {
+ break
+ }
+ sleep(1000)
+ }
+ assertEquals(1, result.size())
+ assertEquals("FINISHED", result[0].State)
+ sql """ insert into ${tbName}(source, report_date) values (1,
'2025-05-12'); """
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
