CalvinKirs commented on code in PR #49541:
URL: https://github.com/apache/doris/pull/49541#discussion_r2055495405
##########
fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java:
##########
@@ -39,32 +40,114 @@
import software.amazon.awssdk.regions.Region;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.sts.StsClient;
+import
software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider;
import java.net.URI;
import java.time.Duration;
public class S3Util {
-
- public static S3Client buildS3Client(URI endpoint, String region,
CloudCredential credential,
- boolean isUsePathStyle) {
- AwsCredentialsProvider scp;
+ private static AwsCredentialsProvider
getAwsCredencialsProvider(CloudCredential credential) {
AwsCredentials awsCredential;
+ AwsCredentialsProvider awsCredentialsProvider;
if (!credential.isTemporary()) {
awsCredential =
AwsBasicCredentials.create(credential.getAccessKey(),
credential.getSecretKey());
} else {
awsCredential =
AwsSessionCredentials.create(credential.getAccessKey(),
credential.getSecretKey(),
credential.getSessionToken());
}
+
if (!credential.isWhole()) {
- scp = AwsCredentialsProviderChain.of(
+ awsCredentialsProvider = AwsCredentialsProviderChain.of(
SystemPropertyCredentialsProvider.create(),
EnvironmentVariableCredentialsProvider.create(),
WebIdentityTokenFileCredentialsProvider.create(),
ProfileCredentialsProvider.create(),
InstanceProfileCredentialsProvider.create());
} else {
- scp = StaticCredentialsProvider.create(awsCredential);
+ awsCredentialsProvider =
StaticCredentialsProvider.create(awsCredential);
+ }
+
+ return awsCredentialsProvider;
+ }
+
+ @Deprecated
+ public static S3Client buildS3Client(URI endpoint, String region,
CloudCredential credential,
+ boolean isUsePathStyle) {
+ EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
+ .builder()
+ .baseDelay(Duration.ofSeconds(1))
+ .maxBackoffTime(Duration.ofMinutes(1))
+ .build();
+ // retry 3 time with Equal backoff
+ RetryPolicy retryPolicy = RetryPolicy
+ .builder()
+ .numRetries(3)
+ .backoffStrategy(backoffStrategy)
+ .build();
+ ClientOverrideConfiguration clientConf = ClientOverrideConfiguration
+ .builder()
+ // set retry policy
+ .retryPolicy(retryPolicy)
+ // using AwsS3V4Signer
+ .putAdvancedOption(SdkAdvancedClientOption.SIGNER,
AwsS3V4Signer.create())
+ .build();
+ return S3Client.builder()
+ .httpClient(UrlConnectionHttpClient.create())
+ .endpointOverride(endpoint)
+ .credentialsProvider(getAwsCredencialsProvider(credential))
+ .region(Region.of(region))
+ .overrideConfiguration(clientConf)
+ // disable chunkedEncoding because of bos not supported
+ .serviceConfiguration(S3Configuration.builder()
+ .chunkedEncodingEnabled(false)
+ .pathStyleAccessEnabled(isUsePathStyle)
+ .build())
+ .build();
+ }
+
+ /**
+ * creating different credentials provider when creating s3client
+ * @param endpoint
+ * @param region
+ * @param accessKey
+ * @param secretKey
+ * @param sessionToken
+ * @param roleArn
+ * @param externalId
+ * @return
+ */
+ private static AwsCredentialsProvider getAwsCredencialsProvider(URI
endpoint, String region, String accessKey,
+ String secretKey, String sessionToken, String roleArn, String
externalId) {
+
+ if (!Strings.isNullOrEmpty(accessKey) &&
!Strings.isNullOrEmpty(secretKey)) {
+ if (Strings.isNullOrEmpty(sessionToken)) {
+ return
StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey,
secretKey));
+ } else {
+ return
StaticCredentialsProvider.create(AwsSessionCredentials.create(accessKey,
+ secretKey, sessionToken));
+ }
+ }
+
+ if (!Strings.isNullOrEmpty(roleArn)) {
+ StsClient stsClient = StsClient.builder()
+
.credentialsProvider(InstanceProfileCredentialsProvider.create())
+ .build();
+ return StsAssumeRoleCredentialsProvider.builder()
+ .stsClient(stsClient)
+ .refreshRequest(r ->
r.roleArn(roleArn).externalId(externalId)
+ .roleSessionName("aws-sdk-java-v2-fe"))
+ .build();
Review Comment:
When a user configures AK/SK/ARN and other parameters simultaneously (or
perhaps we should limit input to only one option?), the actual authentication
method differs from the default AWS behavior, and the behavior seems different
between the BE and FE. It looks like the ARN is actually used in the BE, while
the FE uses AK/SK.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
