luwei16 commented on code in PR #54002:
URL: https://github.com/apache/doris/pull/54002#discussion_r2246864037
##########
cloud/src/recycler/meta_checker.cpp:
##########
@@ -83,175 +60,354 @@ bool MetaChecker::check_fe_meta_by_fdb(MYSQL* conn) {
while (it->has_next()) {
auto [k, v] = it->next();
- TabletIndexPB tablet_idx;
- if (!tablet_idx.ParseFromArray(v.data(), v.size())) [[unlikely]] {
- LOG(WARNING) << "malformed tablet index value";
- return false;
- }
- tablet_indexes.push_back(std::move(tablet_idx));
- if (!it->has_next()) start_key = k;
+ handle_kv(k, v);
+ if (!it->has_next()) {
+ start_key = k;
+ }
}
start_key.push_back('\x00');
} while (it->more());
+ return true;
+}
+bool MetaChecker::do_meta_tablet_key_check(MYSQL* conn) {
+ std::vector<doris::TabletMetaCloudPB> tablets_meta;
bool check_res = true;
+
+ // scan and collect tablet_meta
+ std::string start_key;
+ std::string end_key;
+ meta_tablet_key({instance_id_, 0, 0, 0, 0}, &start_key);
+ meta_tablet_key({instance_id_, INT64_MAX, 0, 0, 0}, &end_key);
+ scan_and_handle_kv(start_key, end_key,
+ [&tablets_meta](std::string_view key, std::string_view
value) -> int {
+ doris::TabletMetaCloudPB tablet_meta;
+ if (!tablet_meta.ParseFromArray(value.data(),
value.size())) {
+ LOG(WARNING) << "malformed tablet meta value";
+ return -1;
+ }
+ tablets_meta.push_back(std::move(tablet_meta));
+ return 0;
+ });
+
+ for (const auto& tablet_meta : tablets_meta) {
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to init txn";
+ continue;
+ }
+
+ // get tablet_index to search tablet belongs which db
+ std::string tablet_index_key;
+ std::string tablet_index_val;
+ meta_tablet_idx_key({instance_id_, tablet_meta.tablet_id()},
&tablet_index_key);
+ err = txn->get(tablet_index_key, &tablet_index_val);
+ if (err != TxnErrorCode::TXN_OK) {
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ LOG(WARNING) << "tablet_idx not found, tablet id: " <<
tablet_meta.tablet_id();
+ continue;
+ } else {
+ LOG(WARNING) << "failed to get tablet_idx, err: " << err
+ << " tablet id: " << tablet_meta.tablet_id();
+ continue;
+ }
+ }
+
+ TabletIndexPB tablet_index_meta;
+ tablet_index_meta.ParseFromString(tablet_index_val);
+
+ if (!db_meta_.contains(tablet_index_meta.db_id())) {
+ LOG(WARNING) << "tablet_idx.db_id not found in fe meta, db_id = "
+ << tablet_index_meta.db_id()
+ << "tablet index meta: " <<
tablet_index_meta.DebugString();
+ continue;
+ }
+ std::string db_name = db_meta_.at(tablet_index_meta.db_id());
+ if (db_name == "__internal_schema" || db_name == "information_schema"
||
+ db_name == "mysql") {
+ continue;
+ }
+
+ if (mysql_select_db(conn, db_name.c_str())) {
+ LOG(WARNING) << "mysql select db error, db_name: " << db_name
+ << " error: " << mysql_error(conn);
+ continue;
+ }
+ MYSQL_RES* result;
+ std::string sql_stmt = "show tablet " +
std::to_string(tablet_meta.tablet_id());
+ mysql_query(conn, sql_stmt.c_str());
+
+ result = mysql_store_result(conn);
+ if (result) {
+ MYSQL_ROW row = mysql_fetch_row(result);
+ auto [db_id, table_id, partition_id, index_id] =
+ std::make_tuple(atoll(row[4]), atoll(row[5]),
atoll(row[6]), atoll(row[7]));
+ if (tablet_meta.table_id() != table_id) {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_meta.ShortDebugString()
+ << " fe table_id: " << atoll(row[5]);
+ check_res = false;
+ }
+ if (tablet_meta.partition_id() != partition_id) {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_meta.ShortDebugString()
+ << " fe partition_id: " << atoll(row[6]);
+ check_res = false;
+ }
+ if (tablet_meta.index_id() != index_id) {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_meta.ShortDebugString()
+ << " fe index_id: " << atoll(row[7]);
+ check_res = false;
+ }
+ mysql_free_result(result);
+ } else {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_meta.ShortDebugString()
+ << " fe tablet not found";
+ check_res = false;
+ }
+ stat_info_.check_fe_tablet_num++;
+ }
+
+ return check_res;
+}
+
+bool MetaChecker::do_meta_tablet_key_index_check(MYSQL* conn) {
+ std::vector<TabletIndexPB> tablet_indexes;
+ bool check_res = true;
+
+ // scan and collect tablet_idx
+ std::string start_key;
+ std::string end_key;
+ meta_tablet_idx_key({instance_id_, 0}, &start_key);
+ meta_tablet_idx_key({instance_id_, INT64_MAX}, &end_key);
+ scan_and_handle_kv(start_key, end_key,
+ [&tablet_indexes](std::string_view key,
std::string_view value) -> int {
+ TabletIndexPB tablet_idx;
+ if (!tablet_idx.ParseFromArray(value.data(),
value.size())) {
+ LOG(WARNING) << "malformed tablet index value";
+ return -1;
+ }
+ tablet_indexes.push_back(std::move(tablet_idx));
+ return 0;
+ });
+
for (const TabletIndexPB& tablet_idx : tablet_indexes) {
+ if (!db_meta_.contains(tablet_idx.db_id())) {
+ LOG(WARNING) << "tablet_idx.db_id not found in fe meta, db_id = "
<< tablet_idx.db_id();
+ continue;
+ }
std::string sql_stmt = "show tablet " +
std::to_string(tablet_idx.tablet_id());
MYSQL_RES* result;
+ std::string db_name = db_meta_.at(tablet_idx.db_id());
+ if (db_name == "__internal_schema" || db_name == "information_schema"
||
+ db_name == "mysql") {
+ continue;
+ }
+ if (mysql_select_db(conn, db_name.c_str())) {
+ LOG(WARNING) << "mysql select db error, db_name: " << db_name
+ << " error: " << mysql_error(conn);
+ continue;
+ }
mysql_query(conn, sql_stmt.c_str());
result = mysql_store_result(conn);
if (result) {
MYSQL_ROW row = mysql_fetch_row(result);
- if (tablet_idx.table_id() != atoll(row[5])) {
+ auto [db_id, table_id, partition_id, index_id] =
+ std::make_tuple(atoll(row[4]), atoll(row[5]),
atoll(row[6]), atoll(row[7]));
+ if (tablet_idx.db_id() != db_id) {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_idx.ShortDebugString()
+ << " fe db_id: " << atoll(row[4]);
+ check_res = false;
+ }
+ if (tablet_idx.table_id() != table_id) {
LOG(WARNING) << "check failed, fdb meta: " <<
tablet_idx.ShortDebugString()
<< " fe table_id: " << atoll(row[5]);
check_res = false;
}
- if (tablet_idx.partition_id() != atoll(row[6])) {
+ if (tablet_idx.partition_id() != partition_id) {
LOG(WARNING) << "check failed, fdb meta: " <<
tablet_idx.ShortDebugString()
<< " fe partition_id: " << atoll(row[6]);
check_res = false;
}
- if (tablet_idx.index_id() != atoll(row[7])) {
+ if (tablet_idx.index_id() != index_id) {
LOG(WARNING) << "check failed, fdb meta: " <<
tablet_idx.ShortDebugString()
<< " fe index_id: " << atoll(row[7]);
check_res = false;
}
+ mysql_free_result(result);
+ } else {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_idx.ShortDebugString()
+ << " fe tablet not found";
+ check_res = false;
}
- mysql_free_result(result);
stat_info_.check_fe_tablet_num++;
}
LOG(INFO) << "check_fe_tablet_num: " << stat_info_.check_fe_tablet_num;
return check_res;
}
-bool MetaChecker::check_fdb_by_fe_meta(MYSQL* conn) {
- // get db info from FE
- std::string sql_stmt = "show databases";
- MYSQL_RES* result;
- mysql_query(conn, sql_stmt.c_str());
- result = mysql_store_result(conn);
- std::map<std::string, std::vector<std::string>*> db_to_tables;
- if (result) {
- int num_row = mysql_num_rows(result);
- for (int i = 0; i < num_row; ++i) {
- MYSQL_ROW row = mysql_fetch_row(result);
- if (strcmp(row[0], "__internal_schema") == 0 ||
- strcmp(row[0], "information_schema") == 0) {
+bool MetaChecker::do_meta_schema_key_check(MYSQL* conn) {
+ std::vector<doris::TabletMetaCloudPB> tablets_meta;
+ bool check_res = true;
+
+ // scan and collect tablet_meta
+ std::string start_key;
+ std::string end_key;
+ meta_tablet_key({instance_id_, 0, 0, 0, 0}, &start_key);
+ meta_tablet_key({instance_id_, INT64_MAX, 0, 0, 0}, &end_key);
+ scan_and_handle_kv(start_key, end_key,
+ [&tablets_meta](std::string_view key, std::string_view
value) -> int {
+ doris::TabletMetaCloudPB tablet_meta;
+ if (!tablet_meta.ParseFromArray(value.data(),
value.size())) {
+ LOG(WARNING) << "malformed tablet meta value";
+ return -1;
+ }
+ tablets_meta.push_back(std::move(tablet_meta));
+ return 0;
+ });
+
+ for (const auto& tablet_meta : tablets_meta) {
+ std::unique_ptr<Transaction> txn;
+ TxnErrorCode err = txn_kv_->create_txn(&txn);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << "failed to init txn";
+ continue;
+ }
+
+ // get tablet_index to search tablet belongs which db
+ std::string tablet_index_key;
+ std::string tablet_index_val;
+ meta_tablet_idx_key({instance_id_, tablet_meta.tablet_id()},
&tablet_index_key);
+ err = txn->get(tablet_index_key, &tablet_index_val);
+ if (err != TxnErrorCode::TXN_OK) {
+ if (err == TxnErrorCode::TXN_KEY_NOT_FOUND) {
+ LOG(WARNING) << "tablet_idx not found, tablet id: " <<
tablet_meta.tablet_id();
+ continue;
+ } else {
+ LOG(WARNING) << "failed to get tablet_idx, err: " << err
+ << " tablet id: " << tablet_meta.tablet_id();
continue;
}
- db_to_tables.insert({row[0], new std::vector<std::string>()});
}
- }
- mysql_free_result(result);
- // get tables info from FE
- for (const auto& elem : db_to_tables) {
- std::string sql_stmt = "show tables from " + elem.first;
+ TabletIndexPB tablet_index_meta;
+ tablet_index_meta.ParseFromString(tablet_index_val);
+
+ if (!db_meta_.contains(tablet_index_meta.db_id())) {
+ LOG(WARNING) << "tablet_idx.db_id not found in fe meta, db_id = "
+ << tablet_index_meta.db_id()
+ << "tablet index meta: " <<
tablet_index_meta.DebugString();
+ continue;
+ }
+ std::string db_name = db_meta_.at(tablet_index_meta.db_id());
+ if (db_name == "__internal_schema" || db_name == "information_schema"
||
+ db_name == "mysql") {
+ continue;
+ }
+
+ if (mysql_select_db(conn, db_name.c_str())) {
+ LOG(WARNING) << "mysql select db error, db_name: " << db_name
+ << " error: " << mysql_error(conn);
+ continue;
+ }
+ std::string tablet_schema_key;
+ std::string tablet_schema_val;
+ meta_schema_key({instance_id_, tablet_index_meta.index_id(),
tablet_meta.schema_version()},
+ &tablet_schema_key);
+ ValueBuf val_buf;
+ err = cloud::blob_get(txn.get(), tablet_schema_key, &val_buf);
+ if (err != TxnErrorCode::TXN_OK) {
+ LOG(WARNING) << fmt::format(
+ "failed to get schema, err={}",
+ err == TxnErrorCode::TXN_KEY_NOT_FOUND ? "not found" :
"internal error");
+ continue;
+ }
+ doris::TabletSchemaCloudPB tablet_schema;
+ if (!parse_schema_value(val_buf, &tablet_schema)) {
+ LOG(WARNING) << fmt::format("malformed schema value, key={}",
tablet_schema_key);
+ continue;
+ }
+
+ MYSQL_RES* result;
+ std::string sql_stmt =
+ fmt::format("SHOW PROC '/dbs/{}/{}/index_schema/{}'",
tablet_index_meta.db_id(),
+ tablet_meta.table_id(), tablet_meta.index_id());
mysql_query(conn, sql_stmt.c_str());
+
result = mysql_store_result(conn);
if (result) {
- int num_row = mysql_num_rows(result);
- for (int i = 0; i < num_row; ++i) {
- MYSQL_ROW row = mysql_fetch_row(result);
- elem.second->push_back(row[0]);
- }
- }
- mysql_free_result(result);
- }
-
- // get tablet info from FE
- std::vector<TabletInfo> tablets;
- for (const auto& elem : db_to_tables) {
- for (const std::string& table : *elem.second) {
- std::string sql_stmt = "show tablets from " + elem.first + "." +
table;
- mysql_query(conn, sql_stmt.c_str());
- result = mysql_store_result(conn);
- if (result) {
- int num_row = mysql_num_rows(result);
- for (int i = 0; i < num_row; ++i) {
- MYSQL_ROW row = mysql_fetch_row(result);
- TabletInfo tablet_info = {0};
- tablet_info.tablet_id = atoll(row[0]);
- VLOG_DEBUG << "get tablet info log"
- << ", db name" << elem.first << ", table name"
<< table
- << ",tablet id" << tablet_info.tablet_id;
- tablet_info.schema_version = atoll(row[4]);
- tablets.push_back(std::move(tablet_info));
- }
+ MYSQL_ROW row = mysql_fetch_row(result);
+ int64_t schema_version = atoll(row[2]);
+ if (tablet_schema.schema_version() != schema_version) {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_schema.ShortDebugString()
+ << " fe schema_version: " << schema_version;
+ check_res = false;
}
mysql_free_result(result);
+ } else {
+ LOG(WARNING) << "check failed, fdb meta: " <<
tablet_meta.ShortDebugString()
+ << " fe tablet not found";
+ check_res = false;
}
+ stat_info_.check_fe_tablet_num++;
}
- // get tablet info from FE
- // get Partition info from FE
- std::map<int64_t, PartitionInfo> partitions;
- for (auto& tablet_info : tablets) {
- std::string sql_stmt = "show tablet " +
std::to_string(tablet_info.tablet_id);
- mysql_query(conn, sql_stmt.c_str());
- result = mysql_store_result(conn);
- if (result) {
- int num_row = mysql_num_rows(result);
- for (int i = 0; i < num_row; ++i) {
- MYSQL_ROW row = mysql_fetch_row(result);
- tablet_info.db_id = atoll(row[4]);
- tablet_info.table_id = atoll(row[5]);
- tablet_info.partition_id = atoll(row[6]);
- tablet_info.index_id = atoll(row[7]);
+ return check_res;
+}
- PartitionInfo partition_info = {0};
- partition_info.db_id = atoll(row[4]);
- partition_info.table_id = atoll(row[5]);
- partition_info.partition_id = atoll(row[6]);
- partition_info.tablet_id = tablet_info.tablet_id;
- VLOG_DEBUG << "get partition info log"
- << ", db id" << partition_info.db_id << ", table id"
- << partition_info.table_id << ", partition id"
- << partition_info.partition_id << ", tablet id"
- << partition_info.tablet_id;
+template <>
+bool MetaChecker::handle_check_fe_meta_by_fdb<CHECK_META>(MYSQL* conn) {
+ bool check_res = true;
+ // check MetaTabletIdxKey
+ if (!do_meta_tablet_key_index_check(conn)) {
+ check_res = false;
+ LOG(WARNING) << "do_meta_tablet_key_index_check failed";
+ } else {
+ LOG(INFO) << "do_meta_tablet_key_index_check success";
+ }
- partitions.insert({partition_info.partition_id,
std::move(partition_info)});
- }
- }
- mysql_free_result(result);
+ // check MetaTabletKey
+ if (!do_meta_tablet_key_check(conn)) {
+ check_res = false;
+ LOG(WARNING) << "do_meta_tablet_key_check failed";
+ } else {
+ LOG(INFO) << "do_meta_tablet_key_check success";
}
- // get partition version from FE
- for (const auto& elem : db_to_tables) {
- for (const std::string& table : *elem.second) {
- std::string sql_stmt = "show partitions from " + elem.first + "."
+ table;
- mysql_query(conn, sql_stmt.c_str());
- result = mysql_store_result(conn);
- if (result) {
- int num_row = mysql_num_rows(result);
- for (int i = 0; i < num_row; ++i) {
- MYSQL_ROW row = mysql_fetch_row(result);
- int64_t partition_id = atoll(row[0]);
- int64_t visible_version = atoll(row[2]);
- partitions[partition_id].visible_version = visible_version;
- VLOG_DEBUG << "get partition version log"
- << ", db name" << elem.first << ", table name"
<< table
- << ", raw partition id" << row[0] << ", first
partition id"
- << partition_id << ", db id" <<
partitions[partition_id].db_id
- << ", table id" <<
partitions[partition_id].table_id
- << ", second partition id" <<
partitions[partition_id].partition_id
- << ", tablet id" <<
partitions[partition_id].tablet_id;
- }
- }
- mysql_free_result(result);
- }
+ // check MetaSchemaKey
+ if (!do_meta_schema_key_check(conn)) {
+ check_res = false;
+ LOG(WARNING) << "do_meta_schema_key_check failed";
+ } else {
+ LOG(INFO) << "do_meta_schema_key_check success";
}
+ return check_res;
+}
+bool MetaChecker::check_fe_meta_by_fdb(MYSQL* conn) {
+ bool success = true;
+ if (config::enable_checker_for_meta_key_check) {
+ success = handle_check_fe_meta_by_fdb<CHECK_META>(conn);
+ }
+
+ // TODO(wyxxxcat) add check for version key
+ // if (config::enable_checker_for_version_key_check) {
+ // success = handle_check_fe_meta_by_fdb<CHECK_VERSION>(conn);
+ // }
+ return success;
+}
+
+bool MetaChecker::do_meta_tablet_index_key_inverted_check(MYSQL* conn,
+ const
std::vector<TabletInfo>& tablets) {
+ bool check_res = true;
// check tablet idx
for (const auto& tablet_info : tablets) {
std::unique_ptr<Transaction> txn;
TxnErrorCode err = txn_kv_->create_txn(&txn);
if (err != TxnErrorCode::TXN_OK) {
LOG(WARNING) << "failed to init txn";
- return false;
+ continue;
Review Comment:
If I continue directly, will I not be able to find out if the tablet is lost?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
