This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new efda2c20fc6 branch-3.1: [fix](checker) Txn is old due to scan too much
kv #56419 (#56525)
efda2c20fc6 is described below
commit efda2c20fc635798101b29556725a7414b902c78
Author: Yixuan Wang <[email protected]>
AuthorDate: Sun Sep 28 15:00:00 2025 +0800
branch-3.1: [fix](checker) Txn is old due to scan too much kv #56419
(#56525)
picked from #56419
---
cloud/src/recycler/checker.cpp | 15 +++++++++--
cloud/test/recycler_test.cpp | 57 ++++++++++++++++++++++++++++++++++++++++++
2 files changed, 70 insertions(+), 2 deletions(-)
diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp
index f75d19f4772..91e3dddb26e 100644
--- a/cloud/src/recycler/checker.cpp
+++ b/cloud/src/recycler/checker.cpp
@@ -2029,10 +2029,21 @@ int InstanceChecker::scan_and_handle_kv(
return -1;
}
std::unique_ptr<RangeGetIterator> it;
+ int limit = 10000;
+ TEST_SYNC_POINT_CALLBACK("InstanceChecker:scan_and_handle_kv:limit",
&limit);
do {
- err = txn->get(start_key, end_key, &it);
+ err = txn->get(start_key, end_key, &it, false, limit);
+ TEST_SYNC_POINT_CALLBACK("InstanceChecker:scan_and_handle_kv:get_err",
&err);
+ if (err == TxnErrorCode::TXN_TOO_OLD) {
+ LOG(WARNING) << "failed to get range kv, err=txn too old, "
+ << " now fallback to non snapshot scan";
+ err = txn_kv_->create_txn(&txn);
+ if (err == TxnErrorCode::TXN_OK) {
+ err = txn->get(start_key, end_key, &it);
+ }
+ }
if (err != TxnErrorCode::TXN_OK) {
- LOG(WARNING) << "failed to get tablet idx, ret=" << err;
+ LOG(WARNING) << "internal error, failed to get range kv, err=" <<
err;
return -1;
}
diff --git a/cloud/test/recycler_test.cpp b/cloud/test/recycler_test.cpp
index 03f089e080a..bdbf3da005b 100644
--- a/cloud/test/recycler_test.cpp
+++ b/cloud/test/recycler_test.cpp
@@ -34,6 +34,7 @@
#include <thread>
#include "common/config.h"
+#include "common/defer.h"
#include "common/logging.h"
#include "common/simple_thread_pool.h"
#include "common/util.h"
@@ -6214,4 +6215,60 @@ TEST(RecyclerTest,
recycle_txn_label_deal_with_conflict_error_test) {
EXPECT_GT(txn_conflict_count, 0) << "txn_conflict sync point should be
triggered";
}
+TEST(CheckerTest, CheckCostTooMuchTime) {
+ DORIS_CLOUD_DEFER {
+ SyncPoint::get_instance()->clear_all_call_backs();
+ };
+
+ InstanceInfoPB instance;
+ instance.set_instance_id(instance_id);
+ auto obj_info = instance.add_obj_info();
+ obj_info->set_id("1");
+
+ auto txn_kv = std::make_shared<MemTxnKv>();
+ ASSERT_EQ(txn_kv->init(), 0);
+ auto sp = SyncPoint::get_instance();
+ sp->enable_processing();
+
+ int64_t tablet_id = 1000;
+ constexpr size_t NUM_BATCH_SIZE = 100;
+ InstanceChecker checker(txn_kv, instance_id);
+ ASSERT_EQ(checker.init(instance), 0);
+
+ for (size_t i = 0; i < NUM_BATCH_SIZE * 2; i++) {
+ std::string key = meta_rowset_key({instance_id, tablet_id, i});
+ std::string value = "rowset_value_" + std::to_string(i);
+ std::unique_ptr<Transaction> txn;
+ ASSERT_EQ(txn_kv->create_txn(&txn), TxnErrorCode::TXN_OK);
+ txn->put(key, value);
+ ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK);
+ }
+
+ sp->set_call_back("InstanceChecker:scan_and_handle_kv:limit",
+ [&](auto&& args) { *try_any_cast<int*>(args[0]) =
NUM_BATCH_SIZE; });
+
+ std::vector<doris::RowsetMetaCloudPB> rowset_metas;
+ size_t count = 0;
+ sp->set_call_back("InstanceChecker:scan_and_handle_kv:get_err", [&](auto&&
args) {
+ if (++count == 2) {
+ *try_any_cast<TxnErrorCode*>(args[0]) = TxnErrorCode::TXN_TOO_OLD;
+ }
+ });
+
+ std::string begin = meta_rowset_key({instance_id, tablet_id, 0});
+ std::string end = meta_rowset_key({instance_id, tablet_id + 1, 0});
+
+ int ret =
+ checker.scan_and_handle_kv(begin, end, [&](std::string_view key,
std::string_view val) {
+ doris::RowsetMetaCloudPB rowset_meta;
+ if (rowset_meta.ParseFromArray(val.data(), val.size()) != 0) {
+ return -1;
+ }
+ rowset_metas.push_back(rowset_meta);
+ return 0;
+ });
+ ASSERT_EQ(0, ret);
+ ASSERT_EQ(rowset_metas.size(), NUM_BATCH_SIZE * 2);
+}
+
} // namespace doris::cloud
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
