This is an automated email from the ASF dual-hosted git repository. yiguolei pushed a commit to branch branch-2.1 in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.1 by this push: new 4f0365e0bfa [fix](s3) move s3 providers to fe-common to be accessible for jni reader (#35779) 4f0365e0bfa is described below commit 4f0365e0bfa502b3237c1ddd89830480b13a5c4c Author: Ashin Gau <ashin...@users.noreply.github.com> AuthorDate: Mon Jun 3 14:04:39 2024 +0800 [fix](s3) move s3 providers to fe-common to be accessible for jni reader (#35779) backport: #35690 `PropertyConverter.setS3FsAccess` has add customized s3 providers: ``` public static final List<String> AWS_CREDENTIALS_PROVIDERS = Arrays.asList( DataLakeAWSCredentialsProvider.class.getName(), TemporaryAWSCredentialsProvider.class.getName(), SimpleAWSCredentialsProvider.class.getName(), EnvironmentVariableCredentialsProvider.class.getName(), IAMInstanceCredentialsProvider.class.getName()); ``` And these providers are set as configuration value of `fs.s3a.aws.credentials.provider`, which will be used as configuration to build s3 reader in JNI readers. However, `DataLakeAWSCredentialsProvider` is in `fe-core`, that is not dependent by JNI readers, so we have to move s3 providers to `fe-common'. --- fe/be-java-extensions/avro-scanner/pom.xml | 31 ---------------------- fe/fe-common/pom.xml | 8 ++++++ .../doris/common}/credentials/CloudCredential.java | 2 +- .../credentials/CloudCredentialWithEndpoint.java | 2 +- .../DataLakeAWSCredentialsProvider.java | 2 +- fe/fe-core/pom.xml | 4 --- .../java/org/apache/doris/catalog/S3Resource.java | 2 +- .../java/org/apache/doris/common/util/S3Util.java | 2 +- .../doris/datasource/iceberg/dlf/DLFCatalog.java | 2 +- .../maxcompute/MaxComputeExternalCatalog.java | 2 +- .../datasource/property/PropertyConverter.java | 4 +-- .../property/constants/BaseProperties.java | 2 +- .../property/constants/CosProperties.java | 2 +- .../property/constants/DLFProperties.java | 2 +- .../property/constants/GCSProperties.java | 2 +- .../property/constants/GlueProperties.java | 2 +- .../property/constants/MCProperties.java | 2 +- .../property/constants/MinioProperties.java | 2 +- .../property/constants/ObsProperties.java | 2 +- .../property/constants/OssProperties.java | 2 +- .../property/constants/PaimonProperties.java | 2 +- .../property/constants/S3Properties.java | 6 ++--- .../java/org/apache/doris/fs/obj/S3ObjStorage.java | 2 +- .../doris/tablefunction/S3TableValuedFunction.java | 2 +- 24 files changed, 32 insertions(+), 59 deletions(-) diff --git a/fe/be-java-extensions/avro-scanner/pom.xml b/fe/be-java-extensions/avro-scanner/pom.xml index 878e4a33e02..8d56a53a7d2 100644 --- a/fe/be-java-extensions/avro-scanner/pom.xml +++ b/fe/be-java-extensions/avro-scanner/pom.xml @@ -56,37 +56,6 @@ under the License. <artifactId>hadoop-common</artifactId> <scope>provided</scope> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-aws</artifactId> - <exclusions> - <exclusion> - <artifactId>slf4j-log4j12</artifactId> - <groupId>org.slf4j</groupId> - </exclusion> - <exclusion> - <artifactId>log4j</artifactId> - <groupId>log4j</groupId> - </exclusion> - <exclusion> - <artifactId>servlet-api</artifactId> - <groupId>javax.servlet</groupId> - </exclusion> - <exclusion> - <artifactId>aws-java-sdk-s3</artifactId> - <groupId>com.amazonaws</groupId> - </exclusion> - <exclusion> - <artifactId>aws-java-sdk-bundle</artifactId> - <groupId>com.amazonaws</groupId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.amazonaws</groupId> - <artifactId>aws-java-sdk-bundle</artifactId> - <version>${aws-java-sdk.version}</version> - </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-hdfs</artifactId> diff --git a/fe/fe-common/pom.xml b/fe/fe-common/pom.xml index c4b1b29b352..04700c02904 100644 --- a/fe/fe-common/pom.xml +++ b/fe/fe-common/pom.xml @@ -108,6 +108,14 @@ under the License. </exclusions> <scope>provided</scope> </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-aws</artifactId> + </dependency> + <dependency> + <groupId>com.amazonaws</groupId> + <artifactId>aws-java-sdk-s3</artifactId> + </dependency> </dependencies> <build> <finalName>doris-fe-common</finalName> diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredential.java b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredential.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredential.java rename to fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredential.java index e80af334105..59e8e44ed27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredential.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredential.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.datasource.credentials; +package org.apache.doris.common.credentials; import org.apache.commons.lang3.StringUtils; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredentialWithEndpoint.java b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredentialWithEndpoint.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredentialWithEndpoint.java rename to fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredentialWithEndpoint.java index 7f80612ea64..085ad030055 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/CloudCredentialWithEndpoint.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/CloudCredentialWithEndpoint.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.datasource.credentials; +package org.apache.doris.common.credentials; public class CloudCredentialWithEndpoint extends CloudCredential { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/DataLakeAWSCredentialsProvider.java b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/DataLakeAWSCredentialsProvider.java similarity index 97% rename from fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/DataLakeAWSCredentialsProvider.java rename to fe/fe-common/src/main/java/org/apache/doris/common/credentials/DataLakeAWSCredentialsProvider.java index 9901b9c6689..600d4440224 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/credentials/DataLakeAWSCredentialsProvider.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/credentials/DataLakeAWSCredentialsProvider.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.datasource.credentials; +package org.apache.doris.common.credentials; import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSCredentials; diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 21a6fac91ff..ef5cd1f5d03 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -386,10 +386,6 @@ under the License. </exclusion> </exclusions> </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-aws</artifactId> - </dependency> <dependency> <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-s3</artifactId> diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 8b9b5f6af37..e1cde40c4ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -20,9 +20,9 @@ package org.apache.doris.catalog; import org.apache.doris.backup.Status; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.PrintableMap; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; import org.apache.doris.datasource.property.constants.S3Properties; import org.apache.doris.fs.remote.S3FileSystem; diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java index 57b53627c68..9e5e258a3ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java @@ -17,7 +17,7 @@ package org.apache.doris.common.util; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; import software.amazon.awssdk.auth.credentials.AwsCredentials; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java index ca5ccd5f359..e9c406715c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java @@ -17,8 +17,8 @@ package org.apache.doris.datasource.iceberg.dlf; +import org.apache.doris.common.credentials.CloudCredential; import org.apache.doris.common.util.S3Util; -import org.apache.doris.datasource.credentials.CloudCredential; import org.apache.doris.datasource.iceberg.HiveCompatibleCatalog; import org.apache.doris.datasource.iceberg.dlf.client.DLFCachedClientPool; import org.apache.doris.datasource.property.PropertyConverter; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java index 6c875d0e29a..ce200ef4fbf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalCatalog.java @@ -19,11 +19,11 @@ package org.apache.doris.datasource.maxcompute; import org.apache.doris.common.DdlException; +import org.apache.doris.common.credentials.CloudCredential; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.credentials.CloudCredential; import org.apache.doris.datasource.property.constants.MCProperties; import com.aliyun.odps.Odps; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java index e3ba2bdce27..425ea6cdcfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java @@ -17,12 +17,12 @@ package org.apache.doris.datasource.property; +import org.apache.doris.common.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.util.LocationPath; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.InitCatalogLog.Type; -import org.apache.doris.datasource.credentials.CloudCredential; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.property.constants.CosProperties; import org.apache.doris.datasource.property.constants.DLFProperties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java index ef79886a865..6ea98607df0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/BaseProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java index c806a49c617..bdf99a76fb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/CosProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java index dd0682114d8..d8a3b7d477a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import com.aliyun.datalake.metastore.common.DataLakeConfig; import com.google.common.collect.ImmutableList; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java index 8a42f417a3e..2ab005d020f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GCSProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java index bf30a050997..4e1598fe32f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/GlueProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import com.amazonaws.glue.catalog.util.AWSGlueConfig; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java index e3059cee4d2..de1983f5753 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MCProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Map; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java index a286718ea86..0499f5fd6fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/MinioProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java index 7d95f9679f3..19ff097fc32 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/ObsProperties.java @@ -18,7 +18,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java index d4fa0e1c650..56497748b8b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/OssProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import java.util.Arrays; import java.util.List; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java index 72560614eae..98739847758 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/PaimonProperties.java @@ -17,7 +17,7 @@ package org.apache.doris.datasource.property.constants; -import org.apache.doris.datasource.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredential; import com.google.common.collect.Maps; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java index 747da72a95b..1dfea17be6f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/S3Properties.java @@ -19,9 +19,9 @@ package org.apache.doris.datasource.property.constants; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; -import org.apache.doris.datasource.credentials.CloudCredential; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; -import org.apache.doris.datasource.credentials.DataLakeAWSCredentialsProvider; +import org.apache.doris.common.credentials.CloudCredential; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; +import org.apache.doris.common.credentials.DataLakeAWSCredentialsProvider; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.thrift.TS3StorageParam; diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java index 11ec72923d0..2e5c9711a01 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java @@ -20,9 +20,9 @@ package org.apache.doris.fs.obj; import org.apache.doris.backup.Status; import org.apache.doris.common.DdlException; import org.apache.doris.common.UserException; +import org.apache.doris.common.credentials.CloudCredential; import org.apache.doris.common.util.S3URI; import org.apache.doris.common.util.S3Util; -import org.apache.doris.datasource.credentials.CloudCredential; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.constants.S3Properties; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 8476f1c978b..196496caf7b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -22,8 +22,8 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; +import org.apache.doris.common.credentials.CloudCredentialWithEndpoint; import org.apache.doris.common.util.S3URI; -import org.apache.doris.datasource.credentials.CloudCredentialWithEndpoint; import org.apache.doris.datasource.property.PropertyConverter; import org.apache.doris.datasource.property.S3ClientBEProperties; import org.apache.doris.datasource.property.constants.S3Properties; --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org