This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-2.1
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c5ffeff833060add948eb98cea7fa538b9fcd918
Author: ryanzryu <143597717+ryanz...@users.noreply.github.com>
AuthorDate: Fri Mar 15 22:19:58 2024 +0800
[fix](s3 client)add default ca cert list for s3 client to avoid
problem:'curlCode:77' (#32285)
Co-authored-by: ryanzryu <ryanz...@tencent.com>
---
be/src/common/config.cpp | 6 ++++++
be/src/common/config.h | 3 +++
be/src/util/s3_util.cpp | 23 +++++++++++++++++++++++
be/src/util/s3_util.h | 2 ++
4 files changed, 34 insertions(+)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index 3d44528ea36..63ed49eca24 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1190,6 +1190,12 @@ DEFINE_mBool(check_segment_when_build_rowset_meta,
"false");
DEFINE_mInt32(max_s3_client_retry, "10");
+// ca_cert_file is in this path by default, Normally no modification is
required
+// ca cert default path is different from different OS
+DEFINE_mString(ca_cert_file_paths,
+
"/etc/pki/tls/certs/ca-bundle.crt;/etc/ssl/certs/ca-certificates.crt;"
+ "/etc/ssl/ca-bundle.pem");
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index a5219c42590..a0a20450d5d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1267,6 +1267,9 @@ DECLARE_mInt32(max_s3_client_retry);
// write as inverted index tmp directory
DECLARE_String(tmp_file_dir);
+// the file paths(one or more) of CA cert, splite using ";" aws s3 lib use it
to init s3client
+DECLARE_mString(ca_cert_file_paths);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index d09b808868b..063cc16c67b 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -30,6 +30,7 @@
#include <atomic>
#include <cstdlib>
+#include <filesystem>
#include <functional>
#include <memory>
#include <ostream>
@@ -114,6 +115,18 @@ S3ClientFactory::S3ClientFactory() {
return std::make_shared<DorisAWSLogger>(logLevel);
};
Aws::InitAPI(_aws_options);
+ _ca_cert_file_path = get_valid_ca_cert_path();
+}
+
+string S3ClientFactory::get_valid_ca_cert_path() {
+ vector<std::string> vec_ca_file_path =
doris::split(config::ca_cert_file_paths, ";");
+ vector<std::string>::iterator it = vec_ca_file_path.begin();
+ for (; it != vec_ca_file_path.end(); ++it) {
+ if (std::filesystem::exists(*it)) {
+ return *it;
+ }
+ }
+ return "";
}
S3ClientFactory::~S3ClientFactory() {
@@ -157,6 +170,16 @@ std::shared_ptr<Aws::S3::S3Client>
S3ClientFactory::create(const S3Conf& s3_conf
Aws::Client::ClientConfiguration aws_config =
S3ClientFactory::getClientConfiguration();
aws_config.endpointOverride = s3_conf.endpoint;
aws_config.region = s3_conf.region;
+ std::string ca_cert = get_valid_ca_cert_path();
+ if ("" != _ca_cert_file_path) {
+ aws_config.caFile = _ca_cert_file_path;
+ } else {
+ // config::ca_cert_file_paths is valmutable,get newest value if file
path invaild
+ _ca_cert_file_path = get_valid_ca_cert_path();
+ if ("" != _ca_cert_file_path) {
+ aws_config.caFile = _ca_cert_file_path;
+ }
+ }
if (s3_conf.max_connections > 0) {
aws_config.maxConnections = s3_conf.max_connections;
} else {
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index a47b5620548..74af0121d20 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -133,10 +133,12 @@ public:
private:
S3ClientFactory();
+ static std::string get_valid_ca_cert_path();
Aws::SDKOptions _aws_options;
std::mutex _lock;
std::unordered_map<uint64_t, std::shared_ptr<Aws::S3::S3Client>> _cache;
+ std::string _ca_cert_file_path;
};
} // end namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
