This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new cf3c3e2104a [feature](Recycler) Retry object storage request when
meeting 429 http error code (#37680)
cf3c3e2104a is described below
commit cf3c3e2104a5a4bc8752f76001c6a22876fbd312
Author: AlexYue <yj976240...@gmail.com>
AuthorDate: Sun Jul 14 23:57:35 2024 +0800
[feature](Recycler) Retry object storage request when meeting 429 http
error code (#37680)
When encountering 429 http code, the client sdk would not do retry by
default. This pr makes the client retry `max_s3_client_retry` times.
---
cloud/src/common/config.h | 3 +++
cloud/src/recycler/s3_accessor.cpp | 32 ++++++++++++++++++++++++++++----
2 files changed, 31 insertions(+), 4 deletions(-)
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 476b79b6752..c5786bedff7 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -196,4 +196,7 @@ CONF_Validator(s3_client_http_scheme, [](const std::string&
config) -> bool {
return config == "http" || config == "https";
});
+// Max retry times for object storage request
+CONF_mInt64(max_s3_client_retry, "10");
+
} // namespace doris::cloud::config
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 1f43f6c6b0e..514baeb7943 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -21,6 +21,7 @@
#include <aws/core/auth/AWSCredentials.h>
#include <aws/core/client/DefaultRetryStrategy.h>
#include <aws/s3/S3Client.h>
+#include <bvar/reducer.h>
#include <gen_cpp/cloud.pb.h>
#include <algorithm>
@@ -41,6 +42,26 @@
#include "recycler/s3_obj_client.h"
#include "recycler/storage_vault_accessor.h"
+namespace {
+
+bvar::Adder<uint64_t>
too_many_request_http_retry_times("too_many_request_http_retry_times");
+
+class CustomRetryStrategy final : public Aws::Client::DefaultRetryStrategy {
+public:
+ CustomRetryStrategy(int maxRetries) : DefaultRetryStrategy(maxRetries) {}
+
+ bool ShouldRetry(const Aws::Client::AWSError<Aws::Client::CoreErrors>&
error,
+ long attemptedRetries) const override {
+ if (attemptedRetries < m_maxRetries &&
+ error.GetResponseCode() ==
Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS) {
+ too_many_request_http_retry_times << 1;
+ return true;
+ }
+ return Aws::Client::DefaultRetryStrategy::ShouldRetry(error,
attemptedRetries);
+ }
+};
+} // namespace
+
namespace doris::cloud {
struct AccessorRateLimiter {
@@ -231,12 +252,15 @@ int S3Accessor::create(S3Conf conf,
std::shared_ptr<S3Accessor>* accessor) {
int S3Accessor::init() {
switch (conf_.provider) {
case S3Conf::AZURE: {
+ Azure::Storage::Blobs::BlobClientOptions options;
+
options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests);
+ options.Retry.MaxRetries = config::max_s3_client_retry;
auto cred =
std::make_shared<Azure::Storage::StorageSharedKeyCredential>(conf_.ak,
conf_.sk);
uri_ = fmt::format("{}://{}.blob.core.windows.net/{}",
config::s3_client_http_scheme,
conf_.ak, conf_.bucket);
- auto container_client =
-
std::make_shared<Azure::Storage::Blobs::BlobContainerClient>(uri_, cred);
+ auto container_client =
std::make_shared<Azure::Storage::Blobs::BlobContainerClient>(
+ uri_, cred, std::move(options));
// uri format for debug:
${scheme}://${ak}.blob.core.windows.net/${bucket}/${prefix}
uri_ = uri_ + '/' + conf_.prefix;
obj_client_ =
std::make_shared<AzureObjClient>(std::move(container_client));
@@ -255,8 +279,8 @@ int S3Accessor::init() {
if (config::s3_client_http_scheme == "http") {
aws_config.scheme = Aws::Http::Scheme::HTTP;
}
- aws_config.retryStrategy =
std::make_shared<Aws::Client::DefaultRetryStrategy>(
- /*maxRetries = 10, scaleFactor = 25*/);
+ aws_config.retryStrategy = std::make_shared<CustomRetryStrategy>(
+ config::max_s3_client_retry /*scaleFactor = 25*/);
auto s3_client = std::make_shared<Aws::S3::S3Client>(
std::move(aws_cred), std::move(aws_config),
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
