This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.1 by this push:
new 378dc724126 branch-4.1: [fix](be): fix BE access to aws under eks pod
identity #67076 (#67555)
378dc724126 is described below
commit 378dc7241264f9a4b9feafebe71a8a41dd6af2e3
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Sep 7 18:25:44 2026 +0800
branch-4.1: [fix](be): fix BE access to aws under eks pod identity #67076
(#67555)
Cherry-picked from #67076
**Depends on #67554** (the branch-4.1 backport of #66851) (the
aws-sdk-cpp 1.11.219 -> 1.11.221 pin). BE cannot compile
this without `GeneralHTTPCredentialsProvider.h`, which first ships in
1.11.221, so that PR has to
land and the pre-built thirdparty artifacts be re-published first.
### Backport notes
branch-4.1 does not have the `common/cpp/obj-client/` refactor, so this
could not be applied
verbatim:
- The `CredProviderType::Container` switch case lives in **two** places
on this branch,
`be/src/util/s3_util.cpp` and `cloud/src/recycler/s3_accessor.cpp`,
instead of the single
`common/cpp/obj-client/auth/aws_credential_factory.cpp` upstream. Both
now call
`create_container_credentials_provider()`, and the then-unused
`<aws/core/platform/Environment.h>` include was dropped from both.
- `common/cpp/aws_common.{h,cpp}` and
`common/cpp/custom_aws_credentials_provider_chain.cpp` carry
the same change as upstream (the new `AWS_CONTAINER_*` constants,
`container_credentials_available()`
and `create_container_credentials_provider()`); the declarations follow
this branch's local
indentation in `aws_common.h`.
- The tests reach the same providers through this branch's entry points:
`S3ClientFactory::get_aws_credentials_provider()` and
`S3Accessor::_get_aws_credentials_provider_v2()`.
- `ContainerProviderTypeIsUsableAsStsBaseProvider` is **not** carried
over. Upstream it exists to
reach `AwsCredentialFactory::create()` below `S3ClientFactory`; this
branch has no such layer, so
the case would have been a second copy of
`ContainerProviderTypeReadsRotatedTokenFileForPodIdentity`
-- both go through the very same
`_create_credentials_provider(Container)`. The other five new
cases, the two new test headers and the MSK IAM cases are all carried
over unchanged.
The other files (`be/src/runtime/aws_msk_iam_auth.cpp`,
`be/test/runtime/aws_msk_iam_auth_test.cpp`,
`gensrc/proto/cloud.proto`, `gensrc/thrift/AgentService.thrift`) applied
cleanly.
`clang-format` 16 was run over every changed file under `be/src`,
`be/test`, `cloud/src` and
`cloud/test` (the paths the Code Formatter workflow checks) and reports
no diff. BE was **not**
compiled locally: this branch's `thirdparty/installed` is still on
aws-sdk-cpp 1.11.219, so the new
header is not available until the backport of #66851 lands.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
https://claude.ai/code/session_01DzUYFcGHQH3bnLGCjpncVj
Co-authored-by: Owen Leung <[email protected]>
---
be/src/runtime/aws_msk_iam_auth.cpp | 5 +-
be/src/util/s3_util.cpp | 4 +-
be/test/io/s3_client_factory_test.cpp | 174 +++++++++++++++++++--
be/test/runtime/aws_msk_iam_auth_test.cpp | 54 +++++++
be/test/testutil/container_credentials_endpoint.h | 99 ++++++++++++
cloud/src/recycler/s3_accessor.cpp | 4 +-
cloud/test/s3_accessor_mock_test.cpp | 74 +++++++++
common/cpp/aws_common.cpp | 58 +++++++
common/cpp/aws_common.h | 32 ++++
.../cpp/custom_aws_credentials_provider_chain.cpp | 50 ++----
common/cpp/test/container_credentials_test_util.h | 131 ++++++++++++++++
gensrc/proto/cloud.proto | 2 +-
gensrc/thrift/AgentService.thrift | 2 +-
13 files changed, 629 insertions(+), 60 deletions(-)
diff --git a/be/src/runtime/aws_msk_iam_auth.cpp
b/be/src/runtime/aws_msk_iam_auth.cpp
index 6d6292b7b01..863d2789a61 100644
--- a/be/src/runtime/aws_msk_iam_auth.cpp
+++ b/be/src/runtime/aws_msk_iam_auth.cpp
@@ -21,7 +21,6 @@
#include <aws/core/auth/AWSCredentialsProvider.h>
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/auth/STSCredentialsProvider.h>
-#include <aws/core/platform/Environment.h>
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
#include <aws/sts/STSClient.h>
#include <aws/sts/model/AssumeRoleRequest.h>
@@ -34,6 +33,7 @@
#include <sstream>
#include "common/logging.h"
+#include "cpp/aws_common.h"
namespace doris {
@@ -51,8 +51,7 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
AwsMskIamAuth::_create_provid
} else if (provider_upper == "INSTANCE_PROFILE" || provider_upper ==
"INSTANCEPROFILE") {
return
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>();
} else if (provider_upper == "CONTAINER" || provider_upper == "ECS") {
- return std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(
-
Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str());
+ return create_container_credentials_provider();
} else if (provider_upper == "SYSTEM_PROPERTIES" || provider_upper ==
"SYSTEMPROPERTIES") {
return
std::make_shared<Aws::Auth::ProfileConfigFileAWSCredentialsProvider>();
} else if (provider_upper == "WEB_IDENTITY" || provider_upper ==
"WEBIDENTITY" ||
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index 1e03308fae4..e3d2ac9be2b 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -22,7 +22,6 @@
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/auth/STSCredentialsProvider.h>
#include <aws/core/client/DefaultRetryStrategy.h>
-#include <aws/core/platform/Environment.h>
#include <aws/core/utils/logging/LogLevel.h>
#include <aws/core/utils/logging/LogSystemInterface.h>
#include <aws/core/utils/memory/stl/AWSStringStream.h>
@@ -356,8 +355,7 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3ClientFactory::_create_cred
case CredProviderType::WebIdentity:
return
std::make_shared<Aws::Auth::STSAssumeRoleWebIdentityCredentialsProvider>();
case CredProviderType::Container:
- return std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(
-
Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str());
+ return create_container_credentials_provider();
case CredProviderType::InstanceProfile:
return
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>();
case CredProviderType::Anonymous:
diff --git a/be/test/io/s3_client_factory_test.cpp
b/be/test/io/s3_client_factory_test.cpp
index de8149380cb..0882fd7847c 100644
--- a/be/test/io/s3_client_factory_test.cpp
+++ b/be/test/io/s3_client_factory_test.cpp
@@ -16,11 +16,14 @@
// under the License.
#include <aws/core/auth/AWSCredentialsProviderChain.h>
+#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
#include <aws/core/auth/STSCredentialsProvider.h>
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
#include <gtest/gtest.h>
+#include <cstdio>
#include <cstdlib>
+#include <memory>
#include <string>
#include <utility>
#include <vector>
@@ -29,6 +32,7 @@
#include "cpp/custom_aws_credentials_provider_chain.h"
#include "io/fs/rate_limited_obj_storage_client.h"
#include "io/fs/s3_obj_storage_client.h"
+#include "testutil/container_credentials_endpoint.h"
#include "util/s3_uri.h"
#include "util/s3_util.h"
@@ -374,16 +378,15 @@ TEST_F(S3ClientFactoryTest,
AwsCredentialsProviderV2ProviderTypeWithoutRoleArn)
provider),
nullptr);
- const char* old_container_uri =
std::getenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI");
- if (old_container_uri == nullptr) {
- setenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI",
"/v2/credentials/mock", 1);
- }
- S3ClientConf container_conf;
- container_conf.cred_provider_type = CredProviderType::Container;
- provider = factory.get_aws_credentials_provider(container_conf);
-
ASSERT_NE(std::dynamic_pointer_cast<Aws::Auth::TaskRoleCredentialsProvider>(provider),
nullptr);
- if (old_container_uri == nullptr) {
- unsetenv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI");
+ {
+ // The environment is pinned rather than merely topped up, so that
this case asserts the
+ // same thing whether or not the machine running it happens to be
inside a container.
+ ContainerCredentialsEnvGuard env;
+ env.set_ecs_task_role("/v2/credentials/mock");
+ S3ClientConf container_conf;
+ container_conf.cred_provider_type = CredProviderType::Container;
+ provider = factory.get_aws_credentials_provider(container_conf);
+ ASSERT_NE(as_valid_http_provider(provider), nullptr);
}
S3ClientConf instance_profile_conf;
@@ -457,4 +460,155 @@ TEST_F(S3ClientFactoryTest,
AwsCredentialsProviderV1RoleArnDefaultFallback) {
config::aws_credentials_provider_version = "v2";
}
+namespace {
+
+// The provider is looked up inside the chain rather than constructed directly,
+// because what is under test is how CustomAwsCredentialsProviderChain wires
the
+// environment.
+Aws::Auth::GeneralHTTPCredentialsProvider* find_http_provider(
+ const CustomAwsCredentialsProviderChain& chain) {
+ for (const auto& provider : chain.GetProviders()) {
+ auto* http_provider =
+
dynamic_cast<Aws::Auth::GeneralHTTPCredentialsProvider*>(provider.get());
+ if (http_provider != nullptr) {
+ return http_provider;
+ }
+ }
+ return nullptr;
+}
+
+} // namespace
+
+// EKS Pod Identity supplies the credential-endpoint token as a file that the
+// kubelet rotates in place, and never as a plain environment variable. The
chain
+// must therefore hand the provider the token path, so that every refresh picks
+// up the current contents. Passing the value read at construction time works
+// until the first rotation and then fails with AccessDenied.
+TEST_F(S3ClientFactoryTest, CustomChainReadsRotatedTokenFileForPodIdentity) {
+ (void)S3ClientFactory::instance();
+
+ ContainerCredentialsEndpoint endpoint;
+ ASSERT_TRUE(endpoint.start());
+
+ ContainerCredentialsEnvGuard env;
+ const std::string token_path = env.token_file_path("custom_chain");
+ env.write_token_file(token_path, "token-one");
+ env.set_pod_identity(endpoint.url(), token_path);
+
+ CustomAwsCredentialsProviderChain chain;
+ auto* provider = find_http_provider(chain);
+ ASSERT_NE(provider, nullptr) << "no GeneralHTTPCredentialsProvider was
added to the chain";
+
+ // Each GetAWSCredentials() drives one HTTP GET to the endpoint: the
provider re-reads the token
+ // file, sends it as the Authorization header, and parses the credentials
from the reply. The
+ // handler records the header it saw, so auth_headers() reports what
actually went over the wire.
+ const auto first_credentials = provider->GetAWSCredentials();
+ EXPECT_EQ(first_credentials.GetAWSAccessKeyId(), "AKIDTEST");
+ EXPECT_EQ(first_credentials.GetSessionToken(), "SESSIONTEST");
+
+ // Rotate the file the way the kubelet does. The second call refetches
rather than serving its
+ // cache only because the handler reports an already-expired Expiration.
+ env.write_token_file(token_path, "token-two");
+ provider->GetAWSCredentials();
+
+ // GE rather than EQ: the credentials client retries and the handler
records every attempt, so an
+ // exact count would make a retry look like a bug. front/back keep the
assertion at full strength
+ // because every request before the rewrite carries token-one and every
one after carries token-two.
+ const auto auth_headers = endpoint.auth_headers();
+ ASSERT_GE(auth_headers.size(), 2u);
+ EXPECT_EQ(auth_headers.front(), "token-one");
+ EXPECT_EQ(auth_headers.back(), "token-two");
+}
+
+// The same wiring has to hold when CONTAINER is asked for by name rather than
reached through the
+// default chain, which is the whole point of a public provider mode. Reading
only
+// AWS_CONTAINER_CREDENTIALS_RELATIVE_URI leaves this provider with an empty
URI, no HTTP client and
+// no credentials on every standard EKS deployment.
+TEST_F(S3ClientFactoryTest,
ContainerProviderTypeReadsRotatedTokenFileForPodIdentity) {
+ S3ClientFactory& factory = S3ClientFactory::instance();
+ config::aws_credentials_provider_version = "v2";
+
+ ContainerCredentialsEndpoint endpoint;
+ ASSERT_TRUE(endpoint.start());
+
+ ContainerCredentialsEnvGuard env;
+ const std::string token_path = env.token_file_path("container_type");
+ env.write_token_file(token_path, "token-one");
+ env.set_pod_identity(endpoint.url(), token_path);
+
+ S3ClientConf conf;
+ conf.cred_provider_type = CredProviderType::Container;
+ auto provider =
as_valid_http_provider(factory.get_aws_credentials_provider(conf));
+ ASSERT_NE(provider, nullptr) << "CONTAINER did not yield a usable
container credentials "
+ "provider for
AWS_CONTAINER_CREDENTIALS_FULL_URI";
+
+ const auto first_credentials = provider->GetAWSCredentials();
+ EXPECT_EQ(first_credentials.GetAWSAccessKeyId(), "AKIDTEST");
+ EXPECT_EQ(first_credentials.GetSessionToken(), "SESSIONTEST");
+
+ env.write_token_file(token_path, "token-two");
+ provider->GetAWSCredentials();
+
+ const auto auth_headers = endpoint.auth_headers();
+ ASSERT_GE(auth_headers.size(), 2u);
+ EXPECT_EQ(auth_headers.front(), "token-one");
+ EXPECT_EQ(auth_headers.back(), "token-two");
+}
+
+TEST_F(S3ClientFactoryTest,
ContainerProviderTypeForwardsInlineAuthorizationToken) {
+ S3ClientFactory& factory = S3ClientFactory::instance();
+ config::aws_credentials_provider_version = "v2";
+
+ ContainerCredentialsEndpoint endpoint;
+ ASSERT_TRUE(endpoint.start());
+
+ ContainerCredentialsEnvGuard env;
+ env.set_inline_token_endpoint(endpoint.url(), "inline-token");
+
+ S3ClientConf conf;
+ conf.cred_provider_type = CredProviderType::Container;
+ auto provider =
as_valid_http_provider(factory.get_aws_credentials_provider(conf));
+ ASSERT_NE(provider, nullptr) << "CONTAINER did not yield a usable
container credentials "
+ "provider for an inline authorization
token";
+
+ EXPECT_EQ(provider->GetAWSCredentials().GetAWSAccessKeyId(), "AKIDTEST");
+
+ const auto auth_headers = endpoint.auth_headers();
+ ASSERT_GE(auth_headers.size(), 1u);
+ EXPECT_EQ(auth_headers.front(), "inline-token");
+}
+
+TEST_F(S3ClientFactoryTest, ContainerProviderTypeStillHonoursEcsRelativeUri) {
+ S3ClientFactory& factory = S3ClientFactory::instance();
+ config::aws_credentials_provider_version = "v2";
+
+ ContainerCredentialsEnvGuard env;
+ env.set_ecs_task_role("/v2/credentials/mock");
+
+ S3ClientConf conf;
+ conf.cred_provider_type = CredProviderType::Container;
+
EXPECT_NE(as_valid_http_provider(factory.get_aws_credentials_provider(conf)),
nullptr)
+ << "CONTAINER did not yield a usable container credentials
provider for "
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
+}
+
+// With neither URI exported there is nothing to talk to, and the provider
says so. This pins the
+// discriminator the two tests above rely on: IsValid() really does
distinguish a wired provider
+// from an inert one, so their assertions cannot pass vacuously.
+TEST_F(S3ClientFactoryTest, ContainerProviderTypeIsUnusableWithoutAnyUri) {
+ S3ClientFactory& factory = S3ClientFactory::instance();
+ config::aws_credentials_provider_version = "v2";
+
+ // Constructing the guard is the whole setup: it clears all four
AWS_CONTAINER_* variables, so
+ // this runs as if on a host that is not a container at all.
+ ContainerCredentialsEnvGuard env;
+
+ S3ClientConf conf;
+ conf.cred_provider_type = CredProviderType::Container;
+ auto provider = factory.get_aws_credentials_provider(conf);
+
ASSERT_NE(std::dynamic_pointer_cast<Aws::Auth::GeneralHTTPCredentialsProvider>(provider),
+ nullptr);
+ EXPECT_EQ(as_valid_http_provider(provider), nullptr);
+}
+
} // namespace doris
diff --git a/be/test/runtime/aws_msk_iam_auth_test.cpp
b/be/test/runtime/aws_msk_iam_auth_test.cpp
index f1eb1ec37ed..954621ece5e 100644
--- a/be/test/runtime/aws_msk_iam_auth_test.cpp
+++ b/be/test/runtime/aws_msk_iam_auth_test.cpp
@@ -17,6 +17,7 @@
#include "runtime/aws_msk_iam_auth.h"
+#include <aws/core/auth/AWSCredentials.h>
#include <gtest/gtest.h>
#include <memory>
@@ -24,6 +25,8 @@
#include <unordered_map>
#include "common/status.h"
+#include "testutil/container_credentials_endpoint.h"
+#include "util/s3_util.h"
namespace doris {
@@ -34,6 +37,14 @@ protected:
config.region = "us-east-1";
}
+ // Nothing in this binary initialises the AWS SDK on its own - neither
BE-UT's main() nor this
+ // file - and without Aws::InitAPI there is no HTTP client factory, so a
credentials provider
+ // quietly returns nothing instead of making a request. Constructing the
S3 client factory is
+ // the cheapest in-tree way to get InitAPI called, exactly once per
process, and it is the same
+ // trick the S3 client factory tests use. Only the tests that drive a real
fetch need it, so it
+ // stays out of SetUp() where it would change what the older tests in this
file do.
+ static void ensure_aws_sdk_initialized() {
(void)S3ClientFactory::instance(); }
+
AwsMskIamAuth::Config config;
};
@@ -215,6 +226,49 @@ TEST_F(AwsMskIamAuthTest,
TestOAuthCallbackCreationWithExternalIdWithoutRoleArn)
ASSERT_EQ(callback, nullptr);
}
+TEST_F(AwsMskIamAuthTest, ContainerProviderReadsTokenFileForPodIdentity) {
+ ensure_aws_sdk_initialized();
+
+ ContainerCredentialsEndpoint endpoint;
+ ASSERT_TRUE(endpoint.start());
+
+ ContainerCredentialsEnvGuard env;
+ const std::string token_path = env.token_file_path("msk_container");
+ env.write_token_file(token_path, "token-one");
+ env.set_pod_identity(endpoint.url(), token_path);
+
+ config.credentials_provider = "CONTAINER";
+ AwsMskIamAuth auth(config);
+
+ Aws::Auth::AWSCredentials credentials;
+ ASSERT_TRUE(auth.get_credentials(&credentials).ok());
+ EXPECT_EQ(credentials.GetAWSAccessKeyId(), "AKIDTEST");
+ EXPECT_EQ(credentials.GetSessionToken(), "SESSIONTEST");
+
+ const auto auth_headers = endpoint.auth_headers();
+ ASSERT_FALSE(auth_headers.empty());
+ EXPECT_EQ(auth_headers.back(), "token-one");
+}
+
+TEST_F(AwsMskIamAuthTest, EcsProviderAliasReachesContainerCredentialsEndpoint)
{
+ ensure_aws_sdk_initialized();
+
+ ContainerCredentialsEndpoint endpoint;
+ ASSERT_TRUE(endpoint.start());
+
+ ContainerCredentialsEnvGuard env;
+ const std::string token_path = env.token_file_path("msk_ecs_alias");
+ env.write_token_file(token_path, "token-one");
+ env.set_pod_identity(endpoint.url(), token_path);
+
+ config.credentials_provider = "ECS";
+ AwsMskIamAuth auth(config);
+
+ Aws::Auth::AWSCredentials credentials;
+ ASSERT_TRUE(auth.get_credentials(&credentials).ok());
+ EXPECT_EQ(credentials.GetAWSAccessKeyId(), "AKIDTEST");
+}
+
// Integration test - only runs if AWS credentials are available
TEST_F(AwsMskIamAuthTest, DISABLED_IntegrationTestWithRealCredentials) {
// This test is disabled by default
diff --git a/be/test/testutil/container_credentials_endpoint.h
b/be/test/testutil/container_credentials_endpoint.h
new file mode 100644
index 00000000000..ac60189a69e
--- /dev/null
+++ b/be/test/testutil/container_credentials_endpoint.h
@@ -0,0 +1,99 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <mutex>
+#include <string>
+#include <vector>
+
+#include "cpp/test/container_credentials_test_util.h"
+#include "service/http/ev_http_server.h"
+#include "service/http/http_channel.h"
+#include "service/http/http_handler.h"
+#include "service/http/http_headers.h"
+#include "service/http/http_method.h"
+#include "service/http/http_request.h"
+
+// Scaffolding for tests of the container credentials providers - the ECS
task-role and EKS Pod
+// Identity endpoints. A credible test needs three awkward pieces at once: a
real HTTP endpoint on an
+// address the AWS SDK is willing to talk to, a token file that can be
rewritten mid-test, and the
+// four AWS_CONTAINER_* environment variables put back however they were found.
+//
+// Only the first of those lives here, because it is the only one the cloud
test tree cannot use: it
+// needs EvHttpServer, which is BE-only. The other two -
ContainerCredentialsEnvGuard and
+// as_valid_http_provider - come from
cpp/test/container_credentials_test_util.h, included above and
+// re-exported to anything that includes this header, so BE tests keep getting
all three from one
+// include.
+namespace doris {
+
+// A mock of the credentials endpoint that ECS and the EKS Pod Identity agent
expose.
+class ContainerCredentialsHandler : public HttpHandler {
+public:
+ void handle(HttpRequest* req) override {
+ {
+ std::lock_guard<std::mutex> lock(_mutex);
+ _auth_headers.push_back(req->header(HttpHeaders::AUTHORIZATION));
+ }
+
+ req->add_output_header(HttpHeaders::CONTENT_TYPE, "application/json");
+ // Expiration in the past keeps ExpiresSoon() true, so the provider
+ // re-reads the token file on the next call instead of serving its
cache.
+ HttpChannel::send_reply(req,
+
R"({"AccessKeyId":"AKIDTEST","SecretAccessKey":"SECRETTEST",)"
+
R"("Token":"SESSIONTEST","Expiration":"1970-01-01T00:00:00Z"})");
+ }
+
+ std::vector<std::string> auth_headers() {
+ std::lock_guard<std::mutex> lock(_mutex);
+ return _auth_headers;
+ }
+
+private:
+ std::mutex _mutex;
+ std::vector<std::string> _auth_headers;
+};
+
+// Starts the mock endpoint on an OS-assigned loopback port and hands out its
URL.
+class ContainerCredentialsEndpoint {
+public:
+ bool start() {
+ if (!_server.register_handler(GET, "/creds", &_handler)) {
+ return false;
+ }
+ _server.start();
+ if (_server.get_real_port() == 0) {
+ return false;
+ }
+ _url = "http://127.0.0.1:"; + std::to_string(_server.get_real_port()) +
"/creds";
+ return true;
+ }
+
+ const std::string& url() const { return _url; }
+
+ std::vector<std::string> auth_headers() { return _handler.auth_headers(); }
+
+private:
+ // _handler is declared before _server so that it is destroyed after it:
~EvHttpServer() stops
+ // the server, and stop() only returns once every worker thread has left
its event loop, so no
+ // request can still be inside _handler by then.
+ ContainerCredentialsHandler _handler;
+ EvHttpServer _server {0};
+ std::string _url;
+};
+
+} // namespace doris
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 7c4fde97041..08effdf17fc 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -22,7 +22,6 @@
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/auth/STSCredentialsProvider.h>
#include <aws/core/client/DefaultRetryStrategy.h>
-#include <aws/core/platform/Environment.h>
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
#include <aws/s3/S3Client.h>
#include <aws/sts/STSClient.h>
@@ -325,8 +324,7 @@ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
S3Accessor::_create_credentia
case CredProviderType::WebIdentity:
return
std::make_shared<Aws::Auth::STSAssumeRoleWebIdentityCredentialsProvider>();
case CredProviderType::Container:
- return std::make_shared<Aws::Auth::TaskRoleCredentialsProvider>(
-
Aws::Environment::GetEnv("AWS_CONTAINER_CREDENTIALS_RELATIVE_URI").c_str());
+ return create_container_credentials_provider();
case CredProviderType::InstanceProfile:
return
std::make_shared<Aws::Auth::InstanceProfileCredentialsProvider>();
case CredProviderType::Anonymous:
diff --git a/cloud/test/s3_accessor_mock_test.cpp
b/cloud/test/s3_accessor_mock_test.cpp
index 5f1e1cc299c..36ceb1da1cc 100644
--- a/cloud/test/s3_accessor_mock_test.cpp
+++ b/cloud/test/s3_accessor_mock_test.cpp
@@ -16,6 +16,7 @@
// under the License.
#include <aws/core/Aws.h>
+#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
#include <aws/s3/S3Client.h>
#include <aws/s3/model/ListObjectsV2Request.h>
#include <aws/s3/model/ListObjectsV2Result.h>
@@ -23,9 +24,16 @@
#include <gmock/gmock.h>
#include <gtest/gtest.h>
+#include <cstdio>
+#include <memory>
+#include <string>
+
#include "common/config.h"
#include "common/logging.h"
+#include "cpp/aws_common.h"
#include "cpp/sync_point.h"
+#include "cpp/test/container_credentials_test_util.h"
+#include "recycler/s3_accessor.h"
#include "recycler/s3_obj_client.h"
using namespace doris;
@@ -83,4 +91,70 @@ TEST_F(S3AccessorMockTest, list_objects_compatibility) {
EXPECT_FALSE(response->is_valid());
}
+namespace {
+
+class ProviderProbe : public S3Accessor {
+public:
+ ProviderProbe() : S3Accessor(S3Conf {}) {}
+
+ // The recycler builds its provider through
_get_aws_credentials_provider_v2(), so the probe
+ // goes through the same call rather than reaching past it. An S3Conf with
no ak/sk and no
+ // role_arn is what a vault configured for container credentials looks
like, and it makes the
+ // factory return the CONTAINER base provider unwrapped.
+ std::shared_ptr<Aws::Auth::AWSCredentialsProvider> container_provider() {
+ S3Conf conf;
+ conf.cred_provider_type = CredProviderType::Container;
+ return _get_aws_credentials_provider_v2(conf);
+ }
+};
+
+} // namespace
+
+// The recycler reaches CredProviderType::Container through a storage vault's
persisted
+// cred_provider_type, and on EKS the pod's credentials live behind
+// AWS_CONTAINER_CREDENTIALS_FULL_URI with the token in a kubelet-rotated
file. Reading only
+// AWS_CONTAINER_CREDENTIALS_RELATIVE_URI leaves this provider with no
endpoint, so recycling silently
+// reclaims nothing.
+TEST_F(S3AccessorMockTest,
container_provider_uses_pod_identity_full_uri_and_token_file) {
+ ContainerCredentialsEnvGuard env;
+ const std::string token_path = env.token_file_path("cloud_pod_identity");
+ env.write_token_file(token_path, "token-one");
+ env.set_pod_identity("http://127.0.0.1:65000/creds";, token_path);
+
+ ProviderProbe probe;
+ EXPECT_NE(as_valid_http_provider(probe.container_provider()), nullptr)
+ << "CONTAINER did not yield a usable container credentials
provider for "
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI";
+
+ ASSERT_EQ(std::remove(token_path.c_str()), 0);
+ EXPECT_EQ(as_valid_http_provider(probe.container_provider()), nullptr)
+ << "CONTAINER ignored AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE: the
provider stayed "
+ "valid "
+ "with the token file removed, so the path was never forwarded";
+}
+
+// ECS is the other half of the same provider mode: forwarding the full URI
must not cost the
+// relative one, which the provider resolves against the ECS agent's own
address.
+TEST_F(S3AccessorMockTest, container_provider_still_honours_ecs_relative_uri) {
+ ContainerCredentialsEnvGuard env;
+ env.set_ecs_task_role("/v2/credentials/mock");
+
+ ProviderProbe probe;
+ EXPECT_NE(as_valid_http_provider(probe.container_provider()), nullptr)
+ << "CONTAINER did not yield a usable container credentials
provider for "
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
+}
+
+// Pins the discriminator the two tests above rely on: with neither URI
exported the provider really
+// is unusable, so their assertions cannot be passing vacuously.
+TEST_F(S3AccessorMockTest, container_provider_is_unusable_without_any_uri) {
+ ContainerCredentialsEnvGuard env;
+
+ ProviderProbe probe;
+ auto provider = probe.container_provider();
+
ASSERT_NE(std::dynamic_pointer_cast<Aws::Auth::GeneralHTTPCredentialsProvider>(provider),
+ nullptr);
+ EXPECT_EQ(as_valid_http_provider(provider), nullptr);
+}
+
} // namespace doris::cloud
diff --git a/common/cpp/aws_common.cpp b/common/cpp/aws_common.cpp
index 3c7f5a0eda5..885f7e7421e 100644
--- a/common/cpp/aws_common.cpp
+++ b/common/cpp/aws_common.cpp
@@ -17,10 +17,17 @@
#include "aws_common.h"
+#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
+#include <aws/core/platform/Environment.h>
+#include <aws/core/utils/memory/AWSMemory.h>
#include <glog/logging.h>
namespace doris {
+namespace {
+const char CONTAINER_CREDENTIALS_PROVIDER_TAG[] =
"ContainerCredentialsProvider";
+} // namespace
+
CredProviderType cred_provider_type_from_pb(cloud::CredProviderTypePB
cred_provider_type) {
switch (cred_provider_type) {
case cloud::CredProviderTypePB::DEFAULT:
@@ -76,6 +83,57 @@ CredProviderType cred_provider_type_from_string(const
std::string& type) {
return CredProviderType::Default;
}
+bool container_credentials_available() {
+ return
!Aws::Environment::GetEnv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI).empty() ||
+
!Aws::Environment::GetEnv(AWS_CONTAINER_CREDENTIALS_FULL_URI).empty();
+}
+
+std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
create_container_credentials_provider() {
+ const auto relative_uri =
Aws::Environment::GetEnv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI);
+ const auto absolute_uri =
Aws::Environment::GetEnv(AWS_CONTAINER_CREDENTIALS_FULL_URI);
+ const auto token =
Aws::Environment::GetEnv(AWS_CONTAINER_AUTHORIZATION_TOKEN);
+ const auto token_path =
Aws::Environment::GetEnv(AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE);
+
+ // Both URIs are forwarded and the provider decides between them: a
non-empty relative URI wins
+ // and is resolved against the ECS agent's address, otherwise the full URI
is used as-is. This
+ // is the same precedence the AWS SDK's own default chain applies.
+ //
+ // Both token forms are forwarded for the same reason. The endpoint
authenticates every fetch
+ // with a bearer token, which the provider takes either inline or as a
file path, and given a
+ // path it re-reads the file before each fetch. ECS sets only the inline
variable, EKS Pod
+ // Identity sets only the file one - so forwarding the path is what makes
the Authorization header
+ // non-empty under Pod Identity, and what keeps it valid once the kubelet
rotates the file.
+ //
+ // NOTE: The header file names its third parameter authTokenFilePath and
its fourth authToken,
+ // but the implementation binds them the other way round. The header is
the side that is wrong,
+ // not the definition. This is reported as aws/aws-sdk-cpp#3143, fixed by
+ // aws/aws-sdk-cpp#3162.
+ auto provider = Aws::MakeShared<Aws::Auth::GeneralHTTPCredentialsProvider>(
+ CONTAINER_CREDENTIALS_PROVIDER_TAG, relative_uri, absolute_uri,
token, token_path);
+
+ const bool uses_relative_uri = !relative_uri.empty();
+ const char* const uri_var = uses_relative_uri ?
AWS_CONTAINER_CREDENTIALS_RELATIVE_URI
+ :
AWS_CONTAINER_CREDENTIALS_FULL_URI;
+ const auto& uri = uses_relative_uri ? relative_uri : absolute_uri;
+
+ if (relative_uri.empty() && absolute_uri.empty()) {
+ LOG(WARNING) << "Container credentials provider has no endpoint to
call and will return no "
+ "credentials: neither "
+ << AWS_CONTAINER_CREDENTIALS_RELATIVE_URI << " nor "
+ << AWS_CONTAINER_CREDENTIALS_FULL_URI << " is set.";
+ } else {
+ LOG(INFO)
+ << "Created container credentials provider from " << uri_var
<< ": [" << uri
+ << "] with a" << (token.empty() ? "n empty" : " non-empty")
+ << " inline authorization token and a"
+ << (token_path.empty() ? "n empty" : " non-empty")
+ << " authorization token file path: [" << token_path
+ << "]. If credentials come back empty, raise aws_log_level to
3 or higher for the "
+ "SDK's own reason.";
+ }
+ return provider;
+}
+
std::string get_valid_ca_cert_path(const std::vector<std::string>&
ca_cert_file_paths) {
for (const auto& path : ca_cert_file_paths) {
if (std::filesystem::exists(path)) {
diff --git a/common/cpp/aws_common.h b/common/cpp/aws_common.h
index cc48e23c2f9..5caff27bec1 100644
--- a/common/cpp/aws_common.h
+++ b/common/cpp/aws_common.h
@@ -20,6 +20,11 @@
#include <gen_cpp/cloud.pb.h>
#include <filesystem>
+#include <memory>
+
+namespace Aws::Auth {
+class AWSCredentialsProvider;
+}
namespace doris {
//AWS Credentials Provider Type
@@ -34,10 +39,37 @@ namespace doris {
Anonymous = 7
};
+ // The environment variables through which a container runtime advertises
its credentials
+ // endpoint. ECS exports a relative URI, which is a path resolved against
the ECS agent's fixed
+ // link-local address, and optionally an inline authorization token. EKS
Pod Identity exports a
+ // full URI, which is a complete URL, and an authorization token file that
the kubelet rotates
+ // in place. A runtime never sets both URIs, so code that reads only one
of them silently gets
+ // nothing on the other platform.
+ inline constexpr char AWS_CONTAINER_CREDENTIALS_RELATIVE_URI[] =
+ "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
+ inline constexpr char AWS_CONTAINER_CREDENTIALS_FULL_URI[] =
+ "AWS_CONTAINER_CREDENTIALS_FULL_URI";
+ inline constexpr char AWS_CONTAINER_AUTHORIZATION_TOKEN[] =
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN";
+ inline constexpr char AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE[] =
+ "AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE";
+
CredProviderType cred_provider_type_from_pb(cloud::CredProviderTypePB
cred_provider_type);
CredProviderType cred_provider_type_from_string(const std::string& type);
+ // Builds the credentials provider that talks to a container runtime's
credentials endpoint,
+ // from the four AWS_CONTAINER_* environment variables above. Every caller
that asks for
+ // container credentials - the default provider chain, and the explicit
CONTAINER/ECS provider
+ // modes of the BE S3 client factory, the cloud recycler and the Kafka MSK
IAM signer - has to
+ // support both ECS and EKS Pod Identity, so none of them can afford to
read a subset of the
+ // variables.
+ std::shared_ptr<Aws::Auth::AWSCredentialsProvider>
create_container_credentials_provider();
+
+ // True when the runtime advertises a container credentials endpoint
through either URI
+ // variable.
+ bool container_credentials_available();
+
std::string get_valid_ca_cert_path(const std::vector<std::string>&
ca_cert_file_paths);
} // namespace doris
\ No newline at end of file
diff --git a/common/cpp/custom_aws_credentials_provider_chain.cpp
b/common/cpp/custom_aws_credentials_provider_chain.cpp
index 5b8ae485bd8..6727136d751 100644
--- a/common/cpp/custom_aws_credentials_provider_chain.cpp
+++ b/common/cpp/custom_aws_credentials_provider_chain.cpp
@@ -18,72 +18,44 @@
#include "custom_aws_credentials_provider_chain.h"
#include <aws/core/auth/AWSCredentialsProviderChain.h>
-#include <aws/core/auth/STSCredentialsProvider.h>
#include <aws/core/auth/SSOCredentialsProvider.h>
+#include <aws/core/auth/STSCredentialsProvider.h>
#include <aws/core/platform/Environment.h>
-#include <aws/core/utils/memory/AWSMemory.h>
#include <aws/core/utils/StringUtils.h>
#include <aws/core/utils/logging/LogMacros.h>
+#include <aws/core/utils/memory/AWSMemory.h>
+
+#include "aws_common.h"
namespace doris {
using namespace Aws::Auth;
using namespace Aws::Utils::Threading;
-static const char AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI[] =
- "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI";
-static const char AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI[] =
"AWS_CONTAINER_CREDENTIALS_FULL_URI";
-static const char AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN[] =
"AWS_CONTAINER_AUTHORIZATION_TOKEN";
static const char AWS_EC2_METADATA_DISABLED[] = "AWS_EC2_METADATA_DISABLED";
static const char DefaultCredentialsProviderChainTag[] =
"DefaultAWSCredentialsProviderChain";
CustomAwsCredentialsProviderChain::CustomAwsCredentialsProviderChain()
: AWSCredentialsProviderChain() {
-
AddProvider(Aws::MakeShared<STSAssumeRoleWebIdentityCredentialsProvider>(
DefaultCredentialsProviderChainTag));
- //ECS TaskRole Credentials only available when ENVIRONMENT VARIABLE is set
- const auto relativeUri =
Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI);
- AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
- "The environment variable value "
- << AWS_ECS_CONTAINER_CREDENTIALS_RELATIVE_URI
<< " is "
- << relativeUri);
-
- const auto absoluteUri =
Aws::Environment::GetEnv(AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI);
- AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
- "The environment variable value " <<
AWS_ECS_CONTAINER_CREDENTIALS_FULL_URI
- << " is " <<
absoluteUri);
-
const auto ec2MetadataDisabled =
Aws::Environment::GetEnv(AWS_EC2_METADATA_DISABLED);
AWS_LOGSTREAM_DEBUG(DefaultCredentialsProviderChainTag,
"The environment variable value " <<
AWS_EC2_METADATA_DISABLED << " is "
<<
ec2MetadataDisabled);
- if (!relativeUri.empty()) {
-
AddProvider(Aws::MakeShared<TaskRoleCredentialsProvider>(DefaultCredentialsProviderChainTag,
-
relativeUri.c_str()));
- AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
- "Added ECS metadata service credentials provider
with relative path: ["
- << relativeUri << "] to the provider
chain.");
- } else if (!absoluteUri.empty()) {
- const auto token =
Aws::Environment::GetEnv(AWS_ECS_CONTAINER_AUTHORIZATION_TOKEN);
- AddProvider(Aws::MakeShared<TaskRoleCredentialsProvider>(
- DefaultCredentialsProviderChainTag, absoluteUri.c_str(),
token.c_str()));
-
- //DO NOT log the value of the authorization token for security
purposes.
- AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
- "Added ECS credentials provider with URI: ["
- << absoluteUri << "] to the provider chain
with a"
- << (token.empty() ? "n empty " : "
non-empty ")
- << "authorization token.");
+ // One provider serves both container platforms:
create_container_credentials_provider() reads
+ // all four AWS_CONTAINER_* variables, applies the provider's own
relative-then-absolute
+ // precedence, and logs which variable supplied the endpoint.
+ if (container_credentials_available()) {
+ AddProvider(create_container_credentials_provider());
}
AddProvider(Aws::MakeShared<InstanceProfileCredentialsProvider>(
DefaultCredentialsProviderChainTag));
- AWS_LOGSTREAM_INFO(
- DefaultCredentialsProviderChainTag,
- "Added EC2 metadata service credentials provider to the provider
chain.");
+ AWS_LOGSTREAM_INFO(DefaultCredentialsProviderChainTag,
+ "Added EC2 metadata service credentials provider to the
provider chain.");
AddProvider(
Aws::MakeShared<EnvironmentAWSCredentialsProvider>(DefaultCredentialsProviderChainTag));
diff --git a/common/cpp/test/container_credentials_test_util.h
b/common/cpp/test/container_credentials_test_util.h
new file mode 100644
index 00000000000..e76d038bc6a
--- /dev/null
+++ b/common/cpp/test/container_credentials_test_util.h
@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#pragma once
+
+#include <aws/core/auth/AWSCredentialsProvider.h>
+#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
+#include <unistd.h>
+
+#include <array>
+#include <cstddef>
+#include <cstdio>
+#include <cstdlib>
+#include <fstream>
+#include <memory>
+#include <optional>
+#include <string>
+#include <vector>
+
+#include "cpp/aws_common.h"
+
+// Test support for the container credentials provider built by
+// create_container_credentials_provider(). It lives in common/cpp/test
because both test trees need
+// it and neither can include the other's: the BE unit tests cover the S3
client factory and the
+// Kafka MSK IAM signer, the cloud unit tests cover the recycler, and all
three reach the same
+// factory.
+namespace doris {
+
+// Clears the four AWS_CONTAINER_* variables on construction and restores them
on destruction.
+class ContainerCredentialsEnvGuard {
+public:
+ ContainerCredentialsEnvGuard() {
+ for (const char* name : kNames) {
+ const char* value = std::getenv(name);
+ _saved.emplace_back(value == nullptr ? std::nullopt
+ :
std::optional<std::string>(value));
+ unsetenv(name);
+ }
+ }
+
+ ~ContainerCredentialsEnvGuard() {
+ for (size_t i = 0; i < kNames.size(); ++i) {
+ if (_saved[i].has_value()) {
+ setenv(kNames[i], _saved[i]->c_str(), 1);
+ } else {
+ unsetenv(kNames[i]);
+ }
+ }
+ for (const auto& path : _token_files) {
+ std::remove(path.c_str());
+ }
+ }
+
+ ContainerCredentialsEnvGuard(const ContainerCredentialsEnvGuard&) = delete;
+ ContainerCredentialsEnvGuard& operator=(const
ContainerCredentialsEnvGuard&) = delete;
+
+ // Presents the environment the way EKS Pod Identity does: a full URL, a
token that exists only
+ // as a file, and no relative URI or inline token at all.
+ void set_pod_identity(const std::string& full_uri, const std::string&
token_file) {
+ unsetenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI);
+ unsetenv(AWS_CONTAINER_AUTHORIZATION_TOKEN);
+ setenv(AWS_CONTAINER_CREDENTIALS_FULL_URI, full_uri.c_str(), 1);
+ setenv(AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE, token_file.c_str(), 1);
+ }
+
+ // Presents the environment the way ECS does: a relative path against the
agent's own address,
+ // and no full URI or token file.
+ void set_ecs_task_role(const std::string& relative_uri) {
+ unsetenv(AWS_CONTAINER_CREDENTIALS_FULL_URI);
+ unsetenv(AWS_CONTAINER_AUTHORIZATION_TOKEN);
+ unsetenv(AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE);
+ setenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI, relative_uri.c_str(),
1);
+ }
+
+ void set_inline_token_endpoint(const std::string& full_uri, const
std::string& token) {
+ unsetenv(AWS_CONTAINER_CREDENTIALS_RELATIVE_URI);
+ unsetenv(AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE);
+ setenv(AWS_CONTAINER_CREDENTIALS_FULL_URI, full_uri.c_str(), 1);
+ setenv(AWS_CONTAINER_AUTHORIZATION_TOKEN, token.c_str(), 1);
+ }
+
+ // Writes a token file and takes responsibility for deleting it. Call it
again with the same path
+ // to rotate the token the way the kubelet does.
+ void write_token_file(const std::string& path, const std::string&
contents) {
+ std::ofstream out(path, std::ios::trunc);
+ out << contents;
+ out.flush();
+ _token_files.push_back(path);
+ }
+
+ // A token file path unique to this test and this process
+ std::string token_file_path(const std::string& name) const {
+ return "/tmp/doris_container_credentials_token_" + name + "_" +
std::to_string(getpid());
+ }
+
+private:
+ static constexpr std::array<const char*, 4> kNames {
+ AWS_CONTAINER_CREDENTIALS_RELATIVE_URI,
AWS_CONTAINER_CREDENTIALS_FULL_URI,
+ AWS_CONTAINER_AUTHORIZATION_TOKEN,
AWS_CONTAINER_AUTHORIZATION_TOKEN_FILE};
+
+ std::vector<std::optional<std::string>> _saved;
+ std::vector<std::string> _token_files;
+};
+
+// Returns the provider only if it is a container credentials provider that
can actually reach an
+// endpoint, and nullptr otherwise.
+inline std::shared_ptr<Aws::Auth::GeneralHTTPCredentialsProvider>
as_valid_http_provider(
+ const std::shared_ptr<Aws::Auth::AWSCredentialsProvider>& provider) {
+ auto http_provider =
+
std::dynamic_pointer_cast<Aws::Auth::GeneralHTTPCredentialsProvider>(provider);
+ if (http_provider != nullptr && !http_provider->IsValid()) {
+ return nullptr;
+ }
+ return http_provider;
+}
+
+} // namespace doris
diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto
index e87bd439efd..3e1b80a8415 100644
--- a/gensrc/proto/cloud.proto
+++ b/gensrc/proto/cloud.proto
@@ -283,7 +283,7 @@ enum CredProviderTypePB {
ENV = 4; // EnvironmentAWSCredentialsProvider
SYSTEM_PROPERTIES = 5; // SystemPropertiesCredentialsProvider
WEB_IDENTITY = 6; // STSAssumeRoleWebIdentityCredentialsProvider
- CONTAINER = 7; // TaskRoleCredentialsProvider
+ CONTAINER = 7; // GeneralHTTPCredentialsProvider, for ECS task roles and
EKS Pod Identity
ANONYMOUS = 8; // AnonymousAWSCredentialsProvider
}
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 32c6c01d335..9973d6fa1f8 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -95,7 +95,7 @@ enum TCredProviderType {
ENV = 3, // EnvironmentAWSCredentialsProvider
SYSTEM_PROPERTIES = 4, // SystemPropertiesCredentialsProvider
WEB_IDENTITY = 5, // STSAssumeRoleWebIdentityCredentialsProvider
- CONTAINER = 6, // TaskRoleCredentialsProvider
+ CONTAINER = 6, // GeneralHTTPCredentialsProvider, for ECS task roles and
EKS Pod Identity
ANONYMOUS = 7 // AnonymousAWSCredentialsProvider
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
