This is an automated email from the ASF dual-hosted git repository.
liaoxin pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new be53e027072 [Enhancement](recycler) Add fault injection config for
s3/azure delete rate limit (#60882)
be53e027072 is described below
commit be53e0270729c67a30ee2549998c7f413883eab1
Author: Yixuan Wang <[email protected]>
AuthorDate: Mon Mar 2 09:32:29 2026 +0800
[Enhancement](recycler) Add fault injection config for s3/azure delete rate
limit (#60882)
- add config flags `enable_s3_rate_limit_inject` and
`s3_rate_limit_inject_probility`
- inject random rate-limit failures on PUT path in S3 and Azure object
clients
- keep existing rate limiter behavior unchanged when injection is
disabled
---
cloud/src/common/config.h | 4 ++++
cloud/src/recycler/azure_obj_client.cpp | 6 ++++++
cloud/src/recycler/s3_obj_client.cpp | 6 ++++++
3 files changed, 16 insertions(+)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index f6aa67a496f..52e7e593dc6 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -245,6 +245,10 @@ CONF_Int32(txn_store_retry_base_intervals_ms, "500");
CONF_Bool(enable_retry_txn_conflict, "true");
CONF_mBool(enable_s3_rate_limiter, "false");
+// Fault injection: randomly return rate limit error for PUT (delete)
operations, for testing recycler.
+// s3_rate_limit_inject_probility is the probability (0-100) of injecting a
rate limit error.
+CONF_mBool(enable_s3_rate_limit_inject, "false");
+CONF_mInt32(s3_rate_limit_inject_probility, "30");
CONF_mInt64(s3_get_bucket_tokens, "1000000000000000000");
CONF_Validator(s3_get_bucket_tokens, [](int64_t config) -> bool { return
config > 0; });
diff --git a/cloud/src/recycler/azure_obj_client.cpp
b/cloud/src/recycler/azure_obj_client.cpp
index b89686622e6..65c0a452403 100644
--- a/cloud/src/recycler/azure_obj_client.cpp
+++ b/cloud/src/recycler/azure_obj_client.cpp
@@ -47,6 +47,12 @@ namespace doris::cloud {
template <typename Func>
auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) {
using T = decltype(callback());
+ // Fault injection for testing rate limit handling in recycler
+ if (config::enable_s3_rate_limit_inject && op == S3RateLimitType::PUT) {
+ if (rand() % 100 < config::s3_rate_limit_inject_probility) {
+ throw std::runtime_error("Azure exceeds request limit");
+ }
+ }
if (!config::enable_s3_rate_limiter) {
return callback();
}
diff --git a/cloud/src/recycler/s3_obj_client.cpp
b/cloud/src/recycler/s3_obj_client.cpp
index fb51f9b3e8d..81d791cabac 100644
--- a/cloud/src/recycler/s3_obj_client.cpp
+++ b/cloud/src/recycler/s3_obj_client.cpp
@@ -46,6 +46,12 @@ namespace doris::cloud {
template <typename Func>
auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) {
using T = decltype(callback());
+ // Fault injection for testing rate limit handling in recycler
+ if (config::enable_s3_rate_limit_inject && op == S3RateLimitType::PUT) {
+ if (rand() % 100 < config::s3_rate_limit_inject_probility) {
+ return T(s3_error_factory());
+ }
+ }
if (!config::enable_s3_rate_limiter) {
return callback();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
