This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new a5d34cfb8a7 branch-3.1: [feat](s3client) Use
`CustomAwsCredentialsProviderChain` for aws sdk #56065 (#56943)
a5d34cfb8a7 is described below
commit a5d34cfb8a73e1c0ab7c96dd7652f7b1d630a56e
Author: Lei Zhang <[email protected]>
AuthorDate: Wed Oct 15 11:51:59 2025 +0800
branch-3.1: [feat](s3client) Use `CustomAwsCredentialsProviderChain` for
aws sdk #56065 (#56943)
picked from #56065
---
be/src/common/config.cpp | 5 +
be/src/common/config.h | 2 +
be/src/util/s3_util.cpp | 51 +++++++++-
be/src/util/s3_util.h | 4 +
be/test/io/s3_client_factory_test.cpp | 106 +++++++++++++++++++++
cloud/src/common/config.h | 3 +
cloud/src/recycler/s3_accessor.cpp | 44 ++++++++-
cloud/src/recycler/s3_accessor.h | 6 ++
.../cpp/custom_aws_credentials_provider_chain.cpp | 103 ++++++++++++++++++++
common/cpp/custom_aws_credentials_provider_chain.h | 30 ++++++
.../main/java/org/apache/doris/common/Config.java | 3 +
.../java/org/apache/doris/common/util/S3Util.java | 64 +++++++++++--
.../datasource/property/storage/S3Properties.java | 48 +++++++++-
.../property/storage/S3PropertiesTest.java | 7 ++
14 files changed, 464 insertions(+), 12 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index fd952b28daa..91d8c021b16 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1577,6 +1577,11 @@ DEFINE_mBool(print_stack_when_cache_miss, "false");
DEFINE_mBool(read_cluster_cache_opt_verbose_log, "false");
+DEFINE_String(aws_credentials_provider_version, "v2");
+DEFINE_Validator(aws_credentials_provider_version, [](const std::string&
config) -> bool {
+ return config == "v1" || config == "v2";
+});
+
// clang-format off
#ifdef BE_TEST
// test s3
diff --git a/be/src/common/config.h b/be/src/common/config.h
index a7152b8d0d0..3c3ec9ac02d 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1643,6 +1643,8 @@ DECLARE_mBool(print_stack_when_cache_miss);
DECLARE_mBool(read_cluster_cache_opt_verbose_log);
+DECLARE_mString(aws_credentials_provider_version);
+
#ifdef BE_TEST
// test s3
DECLARE_String(test_s3_resource);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index d4c4e4e3d34..d16afa4cc82 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -46,6 +46,7 @@
#include "common/logging.h"
#include "common/status.h"
#include "cpp/aws_logger.h"
+#include "cpp/custom_aws_credentials_provider_chain.h"
#include "cpp/obj_retry_strategy.h"
#include "cpp/sync_point.h"
#include "cpp/util.h"
@@ -256,8 +257,8 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_azure_client(
#endif
}
-std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3ClientFactory::get_aws_credentials_provider(
- const S3ClientConf& s3_conf) {
+std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
+S3ClientFactory::_get_aws_credentials_provider_v1(const S3ClientConf& s3_conf)
{
if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
DCHECK(!aws_cred.IsExpiredOrEmpty());
@@ -300,6 +301,52 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3ClientFactory::get_aws_cred
return std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>();
}
+std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
+S3ClientFactory::_get_aws_credentials_provider_v2(const S3ClientConf& s3_conf)
{
+ if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
+ Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
+ DCHECK(!aws_cred.IsExpiredOrEmpty());
+ if (!s3_conf.token.empty()) {
+ aws_cred.SetSessionToken(s3_conf.token);
+ }
+ return
std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(std::move(aws_cred));
+ }
+
+ if (s3_conf.cred_provider_type == CredProviderType::InstanceProfile) {
+ if (s3_conf.role_arn.empty()) {
+ return std::make_shared<CustomAwsCredentialsProviderChain>();
+ }
+
+ Aws::Client::ClientConfiguration clientConfiguration =
+ S3ClientFactory::getClientConfiguration();
+
+ if (_ca_cert_file_path.empty()) {
+ _ca_cert_file_path =
+
get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";"));
+ }
+ if (!_ca_cert_file_path.empty()) {
+ clientConfiguration.caFile = _ca_cert_file_path;
+ }
+
+ auto stsClient = std::make_shared<Aws::STS::STSClient>(
+ std::make_shared<CustomAwsCredentialsProviderChain>(),
clientConfiguration);
+
+ return std::make_shared<Aws::Auth::STSAssumeRoleCredentialsProvider>(
+ s3_conf.role_arn, Aws::String(), s3_conf.external_id,
+ Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient);
+ }
+
+ return std::make_shared<CustomAwsCredentialsProviderChain>();
+}
+
+std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3ClientFactory::get_aws_credentials_provider(
+ const S3ClientConf& s3_conf) {
+ if (config::aws_credentials_provider_version == "v2") {
+ return _get_aws_credentials_provider_v2(s3_conf);
+ }
+ return _get_aws_credentials_provider_v1(s3_conf);
+}
+
std::shared_ptr<io::ObjStorageClient> S3ClientFactory::_create_s3_client(
const S3ClientConf& s3_conf) {
TEST_SYNC_POINT_RETURN_WITH_VALUE(
diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h
index 7f72bd3af26..829cc34e75f 100644
--- a/be/src/util/s3_util.h
+++ b/be/src/util/s3_util.h
@@ -156,6 +156,10 @@ public:
private:
std::shared_ptr<io::ObjStorageClient> _create_s3_client(const
S3ClientConf& s3_conf);
std::shared_ptr<io::ObjStorageClient> _create_azure_client(const
S3ClientConf& s3_conf);
+ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
_get_aws_credentials_provider_v1(
+ const S3ClientConf& s3_conf);
+ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
_get_aws_credentials_provider_v2(
+ const S3ClientConf& s3_conf);
std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
get_aws_credentials_provider(
const S3ClientConf& s3_conf);
diff --git a/be/test/io/s3_client_factory_test.cpp
b/be/test/io/s3_client_factory_test.cpp
new file mode 100644
index 00000000000..0ad6bcae89e
--- /dev/null
+++ b/be/test/io/s3_client_factory_test.cpp
@@ -0,0 +1,106 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
+#include <gtest/gtest.h>
+
+#include "cpp/custom_aws_credentials_provider_chain.h"
+#include "util/s3_util.h"
+
+namespace doris {
+
+class S3ClientFactoryTest : public testing::Test {
+ FRIEND_TEST(S3ClientFactoryTest, S3ClientFactory);
+};
+
+TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) {
+ S3ClientFactory& factory = S3ClientFactory::instance();
+ S3ClientConf anonymous_conf;
+ S3ClientConf ak_sk_conf;
+ ak_sk_conf.ak = "ak";
+ ak_sk_conf.sk = "sk";
+
+ S3ClientConf role_conf1;
+ role_conf1.cred_provider_type = CredProviderType::InstanceProfile;
+
+ S3ClientConf role_conf2;
+ role_conf2.cred_provider_type = CredProviderType::InstanceProfile;
+ role_conf2.role_arn = "role_arn";
+ role_conf2.external_id = "external_id";
+
+ config::aws_credentials_provider_version = "v2";
+ {
+ auto provider_v2 =
factory.get_aws_credentials_provider(anonymous_conf);
+ auto custom_chain_v2 =
+
std::dynamic_pointer_cast<CustomAwsCredentialsProviderChain>(provider_v2);
+ ASSERT_NE(custom_chain_v2, nullptr);
+ }
+ {
+ auto provider_v2 = factory.get_aws_credentials_provider(ak_sk_conf);
+ auto custom_chain_v2 =
+
std::dynamic_pointer_cast<Aws::Auth::SimpleAWSCredentialsProvider>(provider_v2);
+ ASSERT_NE(custom_chain_v2, nullptr);
+ }
+
+ {
+ auto provider_v2 = factory.get_aws_credentials_provider(role_conf1);
+ auto custom_chain_v2 =
+
std::dynamic_pointer_cast<CustomAwsCredentialsProviderChain>(provider_v2);
+ ASSERT_NE(custom_chain_v2, nullptr);
+ }
+
+ {
+ auto provider_v2 = factory.get_aws_credentials_provider(role_conf2);
+ auto custom_chain_v2 =
+
std::dynamic_pointer_cast<Aws::Auth::STSAssumeRoleCredentialsProvider>(provider_v2);
+ ASSERT_NE(custom_chain_v2, nullptr);
+ }
+
+ config::aws_credentials_provider_version = "v1";
+ {
+ auto provider_v1 =
factory.get_aws_credentials_provider(anonymous_conf);
+ auto default_chain_v1 =
+
std::dynamic_pointer_cast<Aws::Auth::AnonymousAWSCredentialsProvider>(provider_v1);
+ ASSERT_NE(default_chain_v1, nullptr);
+ }
+
+ {
+ auto provider_v1 = factory.get_aws_credentials_provider(ak_sk_conf);
+ auto default_chain_v1 =
+
std::dynamic_pointer_cast<Aws::Auth::SimpleAWSCredentialsProvider>(provider_v1);
+ ASSERT_NE(default_chain_v1, nullptr);
+ }
+
+ {
+ auto provider_v1 = factory.get_aws_credentials_provider(role_conf1);
+ auto default_chain_v1 =
+
std::dynamic_pointer_cast<Aws::Auth::InstanceProfileCredentialsProvider>(
+ provider_v1);
+ ASSERT_NE(default_chain_v1, nullptr);
+ }
+
+ {
+ auto provider_v1 = factory.get_aws_credentials_provider(role_conf2);
+ auto default_chain_v1 =
+
std::dynamic_pointer_cast<Aws::Auth::STSAssumeRoleCredentialsProvider>(provider_v1);
+ ASSERT_NE(default_chain_v1, nullptr);
+ }
+
+ config::aws_credentials_provider_version = "v2";
+}
+
+} // namespace doris
\ No newline at end of file
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index 4645c995943..7a576ab54e9 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -347,4 +347,7 @@ CONF_mString(ca_cert_file_paths,
CONF_Bool(enable_check_fe_drop_in_safe_time, "true");
CONF_mBool(enable_logging_conflict_keys, "false");
+CONF_mString(aws_credentials_provider_version, "v2");
+CONF_Validator(aws_credentials_provider_version,
+ [](const std::string& config) -> bool { return config == "v1"
|| config == "v2"; });
} // namespace doris::cloud::config
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 9f8f1ed205e..1f594f868f2 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -45,6 +45,7 @@
#include "common/string_util.h"
#include "common/util.h"
#include "cpp/aws_logger.h"
+#include "cpp/custom_aws_credentials_provider_chain.h"
#include "cpp/obj_retry_strategy.h"
#include "cpp/s3_rate_limiter.h"
#include "cpp/sync_point.h"
@@ -279,7 +280,7 @@ int S3Accessor::create(S3Conf conf,
std::shared_ptr<S3Accessor>* accessor) {
static std::shared_ptr<SimpleThreadPool> worker_pool;
-std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3Accessor::get_aws_credentials_provider(
+std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3Accessor::_get_aws_credentials_provider_v1(
const S3Conf& s3_conf) {
if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
@@ -313,6 +314,47 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3Accessor::get_aws_credentia
return std::make_shared<Aws::Auth::DefaultAWSCredentialsProviderChain>();
}
+std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3Accessor::_get_aws_credentials_provider_v2(
+ const S3Conf& s3_conf) {
+ if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) {
+ Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk);
+ DCHECK(!aws_cred.IsExpiredOrEmpty());
+ return
std::make_shared<Aws::Auth::SimpleAWSCredentialsProvider>(std::move(aws_cred));
+ }
+
+ if (s3_conf.cred_provider_type == CredProviderType::InstanceProfile) {
+ if (s3_conf.role_arn.empty()) {
+ return std::make_shared<CustomAwsCredentialsProviderChain>();
+ }
+
+ Aws::Client::ClientConfiguration clientConfiguration =
+ S3Environment::getClientConfiguration();
+ if (_ca_cert_file_path.empty()) {
+ _ca_cert_file_path =
+
get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';'));
+ }
+ if (!_ca_cert_file_path.empty()) {
+ clientConfiguration.caFile = _ca_cert_file_path;
+ }
+
+ auto stsClient = std::make_shared<Aws::STS::STSClient>(
+ std::make_shared<CustomAwsCredentialsProviderChain>(),
clientConfiguration);
+
+ return std::make_shared<Aws::Auth::STSAssumeRoleCredentialsProvider>(
+ s3_conf.role_arn, Aws::String(), s3_conf.external_id,
+ Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient);
+ }
+ return std::make_shared<CustomAwsCredentialsProviderChain>();
+}
+
+std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3Accessor::get_aws_credentials_provider(
+ const S3Conf& s3_conf) {
+ if (config::aws_credentials_provider_version == "v2") {
+ return _get_aws_credentials_provider_v2(s3_conf);
+ }
+ return _get_aws_credentials_provider_v1(s3_conf);
+}
+
int S3Accessor::init() {
static std::once_flag log_annotated_tags_key_once;
std::call_once(log_annotated_tags_key_once, [&]() {
diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h
index ee72b81e983..5c270184e56 100644
--- a/cloud/src/recycler/s3_accessor.h
+++ b/cloud/src/recycler/s3_accessor.h
@@ -154,6 +154,12 @@ protected:
virtual int delete_prefix_impl(const std::string& path_prefix, int64_t
expiration_time = 0);
+ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
_get_aws_credentials_provider_v1(
+ const S3Conf& s3_conf);
+
+ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
_get_aws_credentials_provider_v2(
+ const S3Conf& s3_conf);
+
std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
get_aws_credentials_provider(
const S3Conf& s3_conf);
diff --git a/common/cpp/custom_aws_credentials_provider_chain.cpp
b/common/cpp/custom_aws_credentials_provider_chain.cpp
new file mode 100644
index 00000000000..b72f97fc0a8
--- /dev/null
+++ b/common/cpp/custom_aws_credentials_provider_chain.cpp
@@ -0,0 +1,103 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "custom_aws_credentials_provider_chain.h"
+
+#include <aws/core/auth/AWSCredentialsProviderChain.h>
+#include <aws/core/auth/STSCredentialsProvider.h>
+#include <aws/core/auth/SSOCredentialsProvider.h>
+#include <aws/core/platform/Environment.h>
+#include <aws/core/utils/memory/AWSMemory.h>
+#include <aws/core/utils/StringUtils.h>
+#include <aws/core/utils/logging/LogMacros.h>
+
+namespace doris {
+
+using namespace Aws::Auth;
+using namespace Aws::Utils::Threading;
+
+static const char AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI[] =
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
+static const char AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI[] =
"AWS_CONTAINER_CREDENTIALS_FULL_URI";
+static const char AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN[] =
"AWS_CONTAINER_AUTHORIZATION_TOKEN";
+static const char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED";
+static const char DefaultCredentialsProviderChainTag[] =
"DefaultAWSCredentialsProviderChain";
+
+CustomAwsCredentialsProviderChain::CustomAwsCredentialsProviderChain()
+ : AWSCredentialsProviderChain() {
+
+ AddProvider(Aws::MakeShared<STSAssumeRoleWebIdentityCredentialsProvider>(
+ DefaultCredentialsProviderChainTag));
+
+ //ECS TaskRole Credentials only available when ENVIRONMENT VARIABLE is set
+ const auto relativeUri =
Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI);
+ AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
+ "The environment variable value "
+ << AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI
<< " is "
+ << relativeUri);
+
+ const auto absoluteUri =
Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI);
+ AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
+ "The environment variable value " <<
AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI
+ << " is " <<
absoluteUri);
+
+ const auto ec2MetadataDisabled =
Aws::Environment::GetEnv(AWS_EC2_METADATA_DISABLED);
+ AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
+ "The environment variable value " <<
AWS_EC2_METADATA_DISABLED << " is "
+ <<
ec2MetadataDisabled);
+
+ if (!relativeUri.empty()) {
+
AddProvider(Aws::MakeShared<TaskRoleCredentialsProvider>(DefaultCredentialsProviderChainTag,
+
relativeUri.c_str()));
+ AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
+ "Added ECS metadata service credentials provider
with relative path: ["
+ << relativeUri << "] to the provider
chain.");
+ } else if (!absoluteUri.empty()) {
+ const auto token =
Aws::Environment::GetEnv(AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN);
+ AddProvider(Aws::MakeShared<TaskRoleCredentialsProvider>(
+ DefaultCredentialsProviderChainTag, absoluteUri.c_str(),
token.c_str()));
+
+ //DO NOT log the value of the authorization token for security
purposes.
+ AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
+ "Added ECS credentials provider with URI: ["
+ << absoluteUri << "] to the provider chain
with a"
+ << (token.empty() ? "n empty " : "
non-empty ")
+ << "authorization token.");
+ }
+
+ AddProvider(Aws::MakeShared<InstanceProfileCredentialsProvider>(
+ DefaultCredentialsProviderChainTag));
+ AWS_LOGSTREAM_INFO(
+ DefaultCredentialsProviderChainTag,
+ "Added EC2 metadata service credentials provider to the provider
chain.");
+
+ AddProvider(
+
Aws::MakeShared<EnvironmentAWSCredentialsProvider>(DefaultCredentialsProviderChainTag));
+ AddProvider(Aws::MakeShared<ProfileConfigFileAWSCredentialsProvider>(
+ DefaultCredentialsProviderChainTag));
+
AddProvider(Aws::MakeShared<ProcessCredentialsProvider>(DefaultCredentialsProviderChainTag));
+
+
AddProvider(Aws::MakeShared<SSOCredentialsProvider>(DefaultCredentialsProviderChainTag));
+}
+
+CustomAwsCredentialsProviderChain::CustomAwsCredentialsProviderChain(
+ const CustomAwsCredentialsProviderChain& chain) {
+ for (const auto& provider : chain.GetProviders()) {
+ AddProvider(provider);
+ }
+}
+}
\ No newline at end of file
diff --git a/common/cpp/custom_aws_credentials_provider_chain.h
b/common/cpp/custom_aws_credentials_provider_chain.h
new file mode 100644
index 00000000000..5f273271f95
--- /dev/null
+++ b/common/cpp/custom_aws_credentials_provider_chain.h
@@ -0,0 +1,30 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <aws/core/auth/AWSCredentialsProviderChain.h>
+
+namespace doris {
+
+class CustomAwsCredentialsProviderChain : public
Aws::Auth::AWSCredentialsProviderChain {
+public:
+ CustomAwsCredentialsProviderChain();
+ CustomAwsCredentialsProviderChain(const CustomAwsCredentialsProviderChain&
chain);
+};
+}
+
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 5276bf0d57a..4e3734480d7 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3478,4 +3478,7 @@ public class Config extends ConfigBase {
"The encryption algorithm used for data, default is AES256, may be set
to empty later for KMS to decide"
})
public static String doris_tde_algorithm = "PLAINTEXT";
+
+ @ConfField(mutable = true)
+ public static String aws_credentials_provider_version = "v2";
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index c147fce6801..f2fc5cda9ec 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -17,6 +17,7 @@
package org.apache.doris.common.util;
+import org.apache.doris.common.Config;
import org.apache.doris.common.credentials.CloudCredential;
import com.google.common.base.Strings;
@@ -25,6 +26,8 @@ import software.amazon.awssdk.auth.credentials.AwsCredentials;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
import software.amazon.awssdk.auth.credentials.AwsSessionCredentials;
+import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
import
software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
@@ -118,7 +121,7 @@ public class S3Util {
* @param externalId AWS External ID for cross-account role assumption
security
* @return
*/
- private static AwsCredentialsProvider getAwsCredencialsProvider(URI
endpoint, String region, String accessKey,
+ private static AwsCredentialsProvider getAwsCredencialsProviderV1(URI
endpoint, String region, String accessKey,
String secretKey, String sessionToken, String roleArn, String
externalId) {
if (!Strings.isNullOrEmpty(accessKey) &&
!Strings.isNullOrEmpty(secretKey)) {
@@ -132,7 +135,7 @@ public class S3Util {
if (!Strings.isNullOrEmpty(roleArn)) {
StsClient stsClient = StsClient.builder()
-
.credentialsProvider(InstanceProfileCredentialsProvider.create())
+ .credentialsProvider(DefaultCredentialsProvider.create())
.build();
return StsAssumeRoleCredentialsProvider.builder()
@@ -144,11 +147,58 @@ public class S3Util {
}
}).build();
}
- return
AwsCredentialsProviderChain.of(SystemPropertyCredentialsProvider.create(),
- EnvironmentVariableCredentialsProvider.create(),
- WebIdentityTokenFileCredentialsProvider.create(),
- ProfileCredentialsProvider.create(),
- InstanceProfileCredentialsProvider.create());
+ return DefaultCredentialsProvider.create();
+ }
+
+ private static AwsCredentialsProvider getAwsCredencialsProviderV2(URI
endpoint, String region, String accessKey,
+ String secretKey, String sessionToken, String roleArn, String
externalId) {
+
+ if (!Strings.isNullOrEmpty(accessKey) &&
!Strings.isNullOrEmpty(secretKey)) {
+ if (Strings.isNullOrEmpty(sessionToken)) {
+ return
StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey,
secretKey));
+ } else {
+ return
StaticCredentialsProvider.create(AwsSessionCredentials.create(accessKey,
+ secretKey, sessionToken));
+ }
+ }
+
+ if (!Strings.isNullOrEmpty(roleArn)) {
+ StsClient stsClient = StsClient.builder()
+ .credentialsProvider(AwsCredentialsProviderChain.of(
+ WebIdentityTokenFileCredentialsProvider.create(),
+ ContainerCredentialsProvider.create(),
+ InstanceProfileCredentialsProvider.create(),
+ SystemPropertyCredentialsProvider.create(),
+ EnvironmentVariableCredentialsProvider.create(),
+ ProfileCredentialsProvider.create()))
+ .build();
+
+ return StsAssumeRoleCredentialsProvider.builder()
+ .stsClient(stsClient)
+ .refreshRequest(builder -> {
+
builder.roleArn(roleArn).roleSessionName("aws-sdk-java-v2-fe");
+ if (!Strings.isNullOrEmpty(externalId)) {
+ builder.externalId(externalId);
+ }
+ }).build();
+ }
+ return AwsCredentialsProviderChain.of(
+ WebIdentityTokenFileCredentialsProvider.create(),
+ ContainerCredentialsProvider.create(),
+ InstanceProfileCredentialsProvider.create(),
+ SystemPropertyCredentialsProvider.create(),
+ EnvironmentVariableCredentialsProvider.create(),
+ ProfileCredentialsProvider.create());
+ }
+
+ private static AwsCredentialsProvider getAwsCredencialsProvider(URI
endpoint, String region, String accessKey,
+ String secretKey, String sessionToken, String roleArn, String
externalId) {
+ if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) {
+ return getAwsCredencialsProviderV2(endpoint, region, accessKey,
secretKey,
+ sessionToken, roleArn, externalId);
+ }
+ return getAwsCredencialsProviderV1(endpoint, region, accessKey,
secretKey,
+ sessionToken, roleArn, externalId);
}
public static S3Client buildS3Client(URI endpoint, String region, boolean
isUsePathStyle,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index e8efd07f22e..e508bff3f6a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -20,6 +20,7 @@ package org.apache.doris.datasource.property.storage;
import org.apache.doris.cloud.proto.Cloud;
import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB;
import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB.Provider;
+import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
import org.apache.doris.datasource.property.ConnectorProperty;
@@ -35,6 +36,7 @@ import org.apache.commons.lang3.StringUtils;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
+import software.amazon.awssdk.auth.credentials.ContainerCredentialsProvider;
import
software.amazon.awssdk.auth.credentials.EnvironmentVariableCredentialsProvider;
import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider;
@@ -285,8 +287,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
}
}
- @Override
- public AwsCredentialsProvider getAwsCredentialsProvider() {
+ private AwsCredentialsProvider getAwsCredentialsProviderV1() {
AwsCredentialsProvider credentialsProvider =
super.getAwsCredentialsProvider();
if (credentialsProvider != null) {
return credentialsProvider;
@@ -318,6 +319,49 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
InstanceProfileCredentialsProvider.create());
}
+ private AwsCredentialsProvider getAwsCredentialsProviderV2() {
+ AwsCredentialsProvider credentialsProvider =
super.getAwsCredentialsProvider();
+ if (credentialsProvider != null) {
+ return credentialsProvider;
+ }
+ if (StringUtils.isNotBlank(s3IAMRole)) {
+ StsClient stsClient = StsClient.builder()
+ .region(Region.of(region))
+ .credentialsProvider(AwsCredentialsProviderChain.of(
+ WebIdentityTokenFileCredentialsProvider.create(),
+ ContainerCredentialsProvider.create(),
+ InstanceProfileCredentialsProvider.create(),
+ SystemPropertyCredentialsProvider.create(),
+ EnvironmentVariableCredentialsProvider.create(),
+ ProfileCredentialsProvider.create()))
+ .build();
+
+ return StsAssumeRoleCredentialsProvider.builder()
+ .stsClient(stsClient)
+ .refreshRequest(builder -> {
+
builder.roleArn(s3IAMRole).roleSessionName("aws-sdk-java-v2-fe");
+ if (StringUtils.isNotBlank(s3ExternalId)) {
+ builder.externalId(s3ExternalId);
+ }
+ }).build();
+ }
+ return AwsCredentialsProviderChain.of(
+ WebIdentityTokenFileCredentialsProvider.create(),
+ ContainerCredentialsProvider.create(),
+ InstanceProfileCredentialsProvider.create(),
+ SystemPropertyCredentialsProvider.create(),
+ EnvironmentVariableCredentialsProvider.create(),
+ ProfileCredentialsProvider.create());
+ }
+
+ @Override
+ public AwsCredentialsProvider getAwsCredentialsProvider() {
+ if (Config.aws_credentials_provider_version.equalsIgnoreCase("v2")) {
+ return getAwsCredentialsProviderV2();
+ }
+ return getAwsCredentialsProviderV1();
+ }
+
@Override
public void initializeHadoopStorageConfig() {
super.initializeHadoopStorageConfig();
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
index f5596957653..a0e94042c59 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/S3PropertiesTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.property.storage;
+import org.apache.doris.common.Config;
import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.UserException;
@@ -28,6 +29,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider;
+import software.amazon.awssdk.auth.credentials.AwsCredentialsProviderChain;
import
software.amazon.awssdk.auth.credentials.InstanceProfileCredentialsProvider;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.services.sts.StsClient;
@@ -410,10 +412,15 @@ public class S3PropertiesTest {
@Test
public void testS3PropertiesAwsAnonymousCredentialsProvider() {
+ Config.aws_credentials_provider_version = "v1";
Map<String, String> props = Maps.newHashMap();
props.put("s3.endpoint", "s3.us-west-2.amazonaws.com");
S3Properties s3Properties = (S3Properties)
StorageProperties.createPrimary(props);
AwsCredentialsProvider provider =
s3Properties.getAwsCredentialsProvider();
Assertions.assertEquals(AnonymousCredentialsProvider.class,
provider.getClass());
+ Config.aws_credentials_provider_version = "v2";
+ provider = s3Properties.getAwsCredentialsProvider();
+ Assertions.assertEquals(AwsCredentialsProviderChain.class,
provider.getClass());
+ Config.aws_credentials_provider_version = "v2";
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
