This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 30aaeeb1131 branch-3.1: [chore](param-refactor)Support independent AWS
connection timeout settings for each object storage #54882 (#54980)
30aaeeb1131 is described below
commit 30aaeeb1131a65eb2ccff589eb3788049dbb5d4f
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Tue Aug 19 19:55:13 2025 +0800
branch-3.1: [chore](param-refactor)Support independent AWS connection
timeout settings for each object storage #54882 (#54980)
Cherry-picked from #54882
Co-authored-by: Calvin Kirs <[email protected]>
---
.../metastore/AbstractIcebergProperties.java | 25 ++++++++-
.../IcebergAliyunDLFMetaStoreProperties.java | 10 ++--
.../IcebergFileSystemMetaStoreProperties.java | 5 +-
.../metastore/IcebergGlueMetaStoreProperties.java | 17 +++---
.../metastore/IcebergHMSMetaStoreProperties.java | 7 +--
.../property/metastore/IcebergRestProperties.java | 3 +-
.../IcebergS3TablesMetaStoreProperties.java | 19 ++-----
.../storage/AbstractS3CompatibleProperties.java | 60 +++-------------------
.../datasource/property/storage/COSProperties.java | 49 ++++++++++++++++++
.../property/storage/MinioProperties.java | 50 ++++++++++++++++++
.../datasource/property/storage/OBSProperties.java | 49 ++++++++++++++++++
.../datasource/property/storage/OSSProperties.java | 49 ++++++++++++++++++
.../property/storage/ObjectStorageProperties.java | 11 ++++
.../datasource/property/storage/S3Properties.java | 29 +++++++++--
.../property/storage/COSPropertiesTest.java | 16 +++---
.../property/storage/OBSPropertyTest.java | 10 ++--
.../property/storage/OSSPropertiesTest.java | 10 ++--
17 files changed, 303 insertions(+), 116 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
index 2aa4fe450f5..359d3786f64 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java
@@ -22,9 +22,11 @@ import
org.apache.doris.datasource.property.ConnectorProperty;
import org.apache.doris.datasource.property.storage.StorageProperties;
import lombok.Getter;
+import org.apache.commons.lang3.StringUtils;
import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.Catalog;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -71,5 +73,26 @@ public abstract class AbstractIcebergProperties extends
MetastoreProperties {
* This field is used to perform metadata operations like creating,
querying,
* and deleting Iceberg tables.
*/
- public abstract Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList);
+ public final Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList) {
+ Map<String, String> catalogProps = new HashMap<>(getOrigProps());
+ if (StringUtils.isNotBlank(warehouse)) {
+ catalogProps.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
+ }
+
+ Catalog catalog = initCatalog(catalogName, catalogProps,
storagePropertiesList);
+
+ if (catalog == null) {
+ throw new IllegalStateException("Catalog must not be null after
initialization.");
+ }
+ return catalog;
+ }
+
+ /**
+ * Subclasses must implement this to create the concrete Catalog instance.
+ */
+ protected abstract Catalog initCatalog(
+ String catalogName,
+ Map<String, String> catalogProps,
+ List<StorageProperties> storagePropertiesList
+ );
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStoreProperties.java
index c1d6124cfd0..26b7149bbc7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergAliyunDLFMetaStoreProperties.java
@@ -23,10 +23,8 @@ import
org.apache.doris.datasource.property.storage.StorageProperties;
import com.aliyun.datalake.metastore.common.DataLakeConfig;
import org.apache.hadoop.conf.Configuration;
-import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.Catalog;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -47,8 +45,8 @@ public class IcebergAliyunDLFMetaStoreProperties extends
AbstractIcebergProperti
}
@Override
- public Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList) {
-
+ public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
+ List<StorageProperties> storagePropertiesList) {
DLFCatalog dlfCatalog = new DLFCatalog();
// @see com.aliyun.datalake.metastore.hive.common.utils.ConfigUtils
Configuration conf = new Configuration();
@@ -63,9 +61,7 @@ public class IcebergAliyunDLFMetaStoreProperties extends
AbstractIcebergProperti
conf.set("hive.metastore.type", "dlf");
conf.set("type", "hms");
dlfCatalog.setConf(conf);
- Map<String, String> catalogProperties = new HashMap<>(origProps);
- catalogProperties.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
- dlfCatalog.initialize(catalogName, catalogProperties);
+ dlfCatalog.initialize(catalogName, catalogProps);
return dlfCatalog;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java
index 995b76c2162..fa94f79df77 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergFileSystemMetaStoreProperties.java
@@ -44,10 +44,9 @@ public class IcebergFileSystemMetaStoreProperties extends
AbstractIcebergPropert
}
@Override
- public Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList) {
+ public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
+ List<StorageProperties> storagePropertiesList) {
Configuration configuration =
buildConfiguration(storagePropertiesList);
- Map<String, String> catalogProps =
buildCatalogProps(storagePropertiesList);
-
HadoopCatalog catalog = new HadoopCatalog();
catalog.setConf(configuration);
try {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
index a544029732f..33a0c774c13 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
@@ -58,20 +58,19 @@ public class IcebergGlueMetaStoreProperties extends
AbstractIcebergProperties {
}
@Override
- public Catalog initializeCatalog(String catalogName,
List<StorageProperties> storageProperties) {
- Map<String, String> props = prepareBaseCatalogProps();
- appendS3Props(props);
- appendGlueProps(props);
- props.put("client.region", glueProperties.glueRegion);
-
+ public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
+ List<StorageProperties> storagePropertiesList) {
+ appendS3Props(catalogProps);
+ appendGlueProps(catalogProps);
+ catalogProps.put("client.region", glueProperties.glueRegion);
if (StringUtils.isNotBlank(warehouse)) {
- props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
+ catalogProps.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
} else {
- props.put(CatalogProperties.WAREHOUSE_LOCATION, CHECKED_WAREHOUSE);
+ catalogProps.put(CatalogProperties.WAREHOUSE_LOCATION,
CHECKED_WAREHOUSE);
}
GlueCatalog catalog = new GlueCatalog();
- catalog.initialize(catalogName, props);
+ catalog.initialize(catalogName, catalogProps);
return catalog;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
index 1ad06e2922a..db9ec672d4a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
@@ -64,12 +64,10 @@ public class IcebergHMSMetaStoreProperties extends
AbstractIcebergProperties {
}
@Override
- public Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList) {
+ public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
+ List<StorageProperties> storagePropertiesList) {
checkInitialized();
-
Configuration conf = buildHiveConfiguration(storagePropertiesList);
- Map<String, String> catalogProps = buildCatalogProperties();
-
HiveCatalog hiveCatalog = new HiveCatalog();
hiveCatalog.setConf(conf);
storagePropertiesList.forEach(sp -> {
@@ -84,7 +82,6 @@ public class IcebergHMSMetaStoreProperties extends
AbstractIcebergProperties {
}
}
});
-
try {
this.executionAuthenticator.execute(() ->
hiveCatalog.initialize(catalogName, catalogProps));
return hiveCatalog;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
index f7ffac84fd0..938bc21ecd2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergRestProperties.java
@@ -161,7 +161,8 @@ public class IcebergRestProperties extends
AbstractIcebergProperties {
}
@Override
- public Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList) {
+ public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
+ List<StorageProperties> storagePropertiesList) {
Map<String, String> fileIOProperties = Maps.newHashMap();
Configuration conf = new Configuration();
toFileIOProperties(storagePropertiesList, fileIOProperties, conf);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
index 4f40bc131e1..3694ab76911 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
@@ -22,12 +22,9 @@ import
org.apache.doris.datasource.iceberg.s3tables.CustomAwsCredentialsProvider
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.iceberg.CatalogProperties;
import org.apache.iceberg.catalog.Catalog;
import software.amazon.s3tables.iceberg.S3TablesCatalog;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -51,14 +48,15 @@ public class IcebergS3TablesMetaStoreProperties extends
AbstractIcebergPropertie
}
@Override
- public Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList) {
+ public Catalog initCatalog(String catalogName, Map<String, String>
catalogProps,
+ List<StorageProperties> storagePropertiesList) {
checkInitialized();
- Map<String, String> props = buildS3CatalogProperties();
+ buildS3CatalogProperties(catalogProps);
S3TablesCatalog catalog = new S3TablesCatalog();
try {
- catalog.initialize(catalogName, props);
+ catalog.initialize(catalogName, catalogProps);
return catalog;
} catch (Exception e) {
throw new RuntimeException("Failed to initialize S3TablesCatalog
for Iceberg. "
@@ -66,19 +64,12 @@ public class IcebergS3TablesMetaStoreProperties extends
AbstractIcebergPropertie
}
}
- private Map<String, String> buildS3CatalogProperties() {
- Map<String, String> props = new HashMap<>();
+ private void buildS3CatalogProperties(Map<String, String> props) {
props.put("client.credentials-provider",
CustomAwsCredentialsProvider.class.getName());
props.put("client.credentials-provider.s3.access-key-id",
s3Properties.getAccessKey());
props.put("client.credentials-provider.s3.secret-access-key",
s3Properties.getSecretKey());
props.put("client.credentials-provider.s3.session-token",
s3Properties.getSessionToken());
props.put("client.region", s3Properties.getRegion());
-
- if (StringUtils.isNotBlank(warehouse)) {
- props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
- }
-
- return props;
}
private void checkInitialized() {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index 509ebd81e87..bd5a4a0824c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -18,11 +18,8 @@
package org.apache.doris.datasource.property.storage;
import org.apache.doris.common.UserException;
-import org.apache.doris.datasource.property.ConnectorProperty;
import com.google.common.base.Strings;
-import lombok.Getter;
-import lombok.Setter;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.logging.log4j.LogManager;
@@ -50,51 +47,6 @@ import java.util.regex.Pattern;
*/
public abstract class AbstractS3CompatibleProperties extends StorageProperties
implements ObjectStorageProperties {
private static final Logger LOG =
LogManager.getLogger(AbstractS3CompatibleProperties.class);
- /**
- * The maximum number of concurrent connections that can be made to the
object storage system.
- * This value is optional and can be configured by the user.
- */
- @Getter
- @ConnectorProperty(names = {"connection.maximum"}, required = false,
description = "Maximum number of connections.")
- protected String maxConnections = "100";
-
- /**
- * The timeout (in milliseconds) for requests made to the object storage
system.
- * This value is optional and can be configured by the user.
- */
- @Getter
- @ConnectorProperty(names = {"connection.request.timeout"}, required =
false,
- description = "Request timeout in seconds.")
- protected String requestTimeoutS = "10000";
-
- /**
- * The timeout (in milliseconds) for establishing a connection to the
object storage system.
- * This value is optional and can be configured by the user.
- */
- @Getter
- @ConnectorProperty(names = {"connection.timeout"}, required = false,
description = "Connection timeout in seconds.")
- protected String connectionTimeoutS = "10000";
-
- /**
- * Flag indicating whether to use path-style URLs for the object storage
system.
- * This value is optional and can be configured by the user.
- */
- @Setter
- @Getter
- @ConnectorProperty(names = {"use_path_style", "s3.path-style-access"},
required = false,
- description = "Whether to use path style URL for the storage.")
- protected String usePathStyle = "false";
- @ConnectorProperty(names = {"force_parsing_by_standard_uri"}, required =
false,
- description = "Whether to use path style URL for the storage.")
- @Setter
- @Getter
- protected String forceParsingByStandardUrl = "false";
-
- @Getter
- @ConnectorProperty(names = {"s3.session_token", "session_token"},
- required = false,
- description = "The session token of S3.")
- protected String sessionToken = "";
/**
* Constructor to initialize the object storage properties with the
provided type and original properties map.
@@ -132,7 +84,8 @@ public abstract class AbstractS3CompatibleProperties extends
StorageProperties i
* @return a map containing AWS S3 configuration properties.
*/
protected Map<String, String> generateBackendS3Configuration() {
- return doBuildS3Configuration(maxConnections, requestTimeoutS,
connectionTimeoutS, usePathStyle);
+ return doBuildS3Configuration(getMaxConnections(),
getRequestTimeoutS(), getConnectionTimeoutS(),
+ getUsePathStyle());
}
/**
@@ -164,11 +117,11 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
public AwsCredentialsProvider getAwsCredentialsProvider() {
if (StringUtils.isNotBlank(getAccessKey()) &&
StringUtils.isNotBlank(getSecretKey())) {
- if (Strings.isNullOrEmpty(sessionToken)) {
+ if (Strings.isNullOrEmpty(getSessionToken())) {
return
StaticCredentialsProvider.create(AwsBasicCredentials.create(getAccessKey(),
getSecretKey()));
} else {
return
StaticCredentialsProvider.create(AwsSessionCredentials.create(getAccessKey(),
getSecretKey(),
- sessionToken));
+ getSessionToken()));
}
}
return null;
@@ -207,7 +160,8 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
String endpoint = null;
// 1. try getting endpoint from uri
try {
- endpoint = S3PropertyUtils.constructEndpointFromUrl(origProps,
usePathStyle, forceParsingByStandardUrl);
+ endpoint = S3PropertyUtils.constructEndpointFromUrl(origProps,
getUsePathStyle(),
+ getForceParsingByStandardUrl());
} catch (Exception e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Failed to construct endpoint from url: " +
origProps, e);
@@ -309,7 +263,7 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
hadoopStorageConfig.set("fs.s3a.connection.maximum",
getMaxConnections());
hadoopStorageConfig.set("fs.s3a.connection.request.timeout",
getRequestTimeoutS());
hadoopStorageConfig.set("fs.s3a.connection.timeout",
getConnectionTimeoutS());
- hadoopStorageConfig.set("fs.s3a.path.style.access", usePathStyle);
+ hadoopStorageConfig.set("fs.s3a.path.style.access", getUsePathStyle());
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
index f1820abb2b3..8e09576891e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
@@ -63,6 +63,55 @@ public class COSProperties extends
AbstractS3CompatibleProperties {
description = "The secret key of COS.")
protected String secretKey = "";
+ @Getter
+ @ConnectorProperty(names = {"cos.session_token", "s3.session_token",
"session_token"},
+ required = false,
+ description = "The session token of COS.")
+ protected String sessionToken = "";
+
+ /**
+ * The maximum number of concurrent connections that can be made to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"cos.connection.maximum",
"s3.connection.maximum"}, required = false,
+ description = "Maximum number of connections.")
+ protected String maxConnections = "100";
+
+ /**
+ * The timeout (in milliseconds) for requests made to the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"cos.connection.request.timeout",
"s3.connection.request.timeout"}, required = false,
+ description = "Request timeout in seconds.")
+ protected String requestTimeoutS = "10000";
+
+ /**
+ * The timeout (in milliseconds) for establishing a connection to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"cos.connection.timeout",
"s3.connection.timeout"}, required = false,
+ description = "Connection timeout in seconds.")
+ protected String connectionTimeoutS = "10000";
+
+ /**
+ * Flag indicating whether to use path-style URLs for the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Setter
+ @Getter
+ @ConnectorProperty(names = {"cos.use_path_style", "use_path_style",
"s3.path-style-access"}, required = false,
+ description = "Whether to use path style URL for the storage.")
+ protected String usePathStyle = "false";
+
+ @ConnectorProperty(names = {"cos.force_parsing_by_standard_uri",
"force_parsing_by_standard_uri"}, required = false,
+ description = "Whether to use path style URL for the storage.")
+ @Setter
+ @Getter
+ protected String forceParsingByStandardUrl = "false";
+
/**
* Pattern to extract the region from a Tencent Cloud COS endpoint.
* <p>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java
index 374807c48bd..6f0496ae5ff 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/MinioProperties.java
@@ -51,6 +51,56 @@ public class MinioProperties extends
AbstractS3CompatibleProperties {
description = "The secret key of Minio.")
protected String secretKey = "";
+ @Getter
+ @ConnectorProperty(names = {"minio.session_token", "s3.session_token",
"session_token"},
+ required = false,
+ description = "The session token of Minio.")
+ protected String sessionToken = "";
+
+ /**
+ * The maximum number of concurrent connections that can be made to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"minio.connection.maximum",
"s3.connection.maximum"}, required = false,
+ description = "Maximum number of connections.")
+ protected String maxConnections = "100";
+
+ /**
+ * The timeout (in milliseconds) for requests made to the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"minio.connection.request.timeout",
"s3.connection.request.timeout"}, required = false,
+ description = "Request timeout in seconds.")
+ protected String requestTimeoutS = "10000";
+
+ /**
+ * The timeout (in milliseconds) for establishing a connection to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"minio.connection.timeout",
"s3.connection.timeout"}, required = false,
+ description = "Connection timeout in seconds.")
+ protected String connectionTimeoutS = "10000";
+
+ /**
+ * Flag indicating whether to use path-style URLs for the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Setter
+ @Getter
+ @ConnectorProperty(names = {"minio.use_path_style", "use_path_style",
"s3.path-style-access"}, required = false,
+ description = "Whether to use path style URL for the storage.")
+ protected String usePathStyle = "false";
+
+ @ConnectorProperty(names = {"minio.force_parsing_by_standard_uri",
"force_parsing_by_standard_uri"},
+ required = false,
+ description = "Whether to use path style URL for the storage.")
+ @Setter
+ @Getter
+ protected String forceParsingByStandardUrl = "false";
+
private static final Set<String> IDENTIFIERS =
ImmutableSet.of("minio.access_key", "AWS_ACCESS_KEY", "ACCESS_KEY",
"access_key", "s3.access_key", "minio.endpoint", "s3.endpoint",
"AWS_ENDPOINT", "endpoint", "ENDPOINT");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
index 4dda2977169..5e9c513d67e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OBSProperties.java
@@ -62,6 +62,55 @@ public class OBSProperties extends
AbstractS3CompatibleProperties {
description = "The region of OBS.")
protected String region;
+ @Getter
+ @ConnectorProperty(names = {"obs.session_token", "s3.session_token",
"session_token"},
+ required = false,
+ description = "The session token of OBS.")
+ protected String sessionToken = "";
+
+ /**
+ * The maximum number of concurrent connections that can be made to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"obs.connection.maximum",
"s3.connection.maximum"}, required = false,
+ description = "Maximum number of connections.")
+ protected String maxConnections = "100";
+
+ /**
+ * The timeout (in milliseconds) for requests made to the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"obs.connection.request.timeout",
"s3.connection.request.timeout"}, required = false,
+ description = "Request timeout in seconds.")
+ protected String requestTimeoutS = "10000";
+
+ /**
+ * The timeout (in milliseconds) for establishing a connection to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"obs.connection.timeout",
"s3.connection.timeout"}, required = false,
+ description = "Connection timeout in seconds.")
+ protected String connectionTimeoutS = "10000";
+
+ /**
+ * Flag indicating whether to use path-style URLs for the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Setter
+ @Getter
+ @ConnectorProperty(names = {"obs.use_path_style", "use_path_style",
"s3.path-style-access"}, required = false,
+ description = "Whether to use path style URL for the storage.")
+ protected String usePathStyle = "false";
+
+ @ConnectorProperty(names = {"obs.force_parsing_by_standard_uri",
"force_parsing_by_standard_uri"}, required = false,
+ description = "Whether to use path style URL for the storage.")
+ @Setter
+ @Getter
+ protected String forceParsingByStandardUrl = "false";
+
/**
* Pattern to extract the region from a Huawei Cloud OBS endpoint.
* <p>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index 07d56648498..4cb43b3155a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -74,6 +74,55 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
description = "Enable public access to Aliyun DLF.")
protected String dlfAccessPublic = "false";
+ @Getter
+ @ConnectorProperty(names = {"oss.session_token", "s3.session_token",
"session_token"},
+ required = false,
+ description = "The session token of OSS.")
+ protected String sessionToken = "";
+
+ /**
+ * The maximum number of concurrent connections that can be made to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"oss.connection.maximum",
"s3.connection.maximum"}, required = false,
+ description = "Maximum number of connections.")
+ protected String maxConnections = "100";
+
+ /**
+ * The timeout (in milliseconds) for requests made to the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"oss.connection.request.timeout",
"s3.connection.request.timeout"}, required = false,
+ description = "Request timeout in seconds.")
+ protected String requestTimeoutS = "10000";
+
+ /**
+ * The timeout (in milliseconds) for establishing a connection to the
object storage system.
+ * This value is optional and can be configured by the user.
+ */
+ @Getter
+ @ConnectorProperty(names = {"oss.connection.timeout",
"s3.connection.timeout"}, required = false,
+ description = "Connection timeout in seconds.")
+ protected String connectionTimeoutS = "10000";
+
+ /**
+ * Flag indicating whether to use path-style URLs for the object storage
system.
+ * This value is optional and can be configured by the user.
+ */
+ @Setter
+ @Getter
+ @ConnectorProperty(names = {"oss.use_path_style", "use_path_style",
"s3.path-style-access"}, required = false,
+ description = "Whether to use path style URL for the storage.")
+ protected String usePathStyle = "false";
+
+ @ConnectorProperty(names = {"oss.force_parsing_by_standard_uri",
"force_parsing_by_standard_uri"}, required = false,
+ description = "Whether to use path style URL for the storage.")
+ @Setter
+ @Getter
+ protected String forceParsingByStandardUrl = "false";
+
/**
* Pattern to extract the region from an Alibaba Cloud OSS endpoint.
* <p>
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
index f36e03e1b25..9add87fd368 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/ObjectStorageProperties.java
@@ -36,4 +36,15 @@ public interface ObjectStorageProperties {
void setRegion(String region);
+ String getSessionToken();
+
+ String getMaxConnections();
+
+ String getRequestTimeoutS();
+
+ String getConnectionTimeoutS();
+
+ String getUsePathStyle();
+
+ String getForceParsingByStandardUrl();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 7985d81ea47..bd05f39d4ab 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -85,24 +85,44 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
description = "The secret key of S3. Optional for anonymous access
to public datasets.")
protected String secretKey = "";
+ @Getter
+ @ConnectorProperty(names = {"s3.session_token", "session_token"},
+ required = false,
+ description = "The session token of S3.")
+ protected String sessionToken = "";
+ @Getter
@ConnectorProperty(names = {"s3.connection.maximum",
"AWS_MAX_CONNECTIONS"},
required = false,
description = "The maximum number of connections to S3.")
- protected String s3ConnectionMaximum = "50";
+ protected String maxConnections = "50";
+ @Getter
@ConnectorProperty(names = {"s3.connection.request.timeout",
"AWS_REQUEST_TIMEOUT_MS"},
required = false,
description = "The request timeout of S3 in milliseconds,")
- protected String s3ConnectionRequestTimeoutS = "3000";
+ protected String requestTimeoutS = "3000";
+ @Getter
@ConnectorProperty(names = {"s3.connection.timeout",
"AWS_CONNECTION_TIMEOUT_MS"},
required = false,
description = "The connection timeout of S3 in milliseconds,")
- protected String s3ConnectionTimeoutS = "1000";
+ protected String connectionTimeoutS = "1000";
+
+ @Setter
+ @Getter
+ @ConnectorProperty(names = {"use_path_style", "s3.path-style-access"},
required = false,
+ description = "Whether to use path style URL for the storage.")
+ protected String usePathStyle = "false";
+
+ @ConnectorProperty(names = {"force_parsing_by_standard_uri"}, required =
false,
+ description = "Whether to use path style URL for the storage.")
+ @Setter
+ @Getter
+ protected String forceParsingByStandardUrl = "false";
@ConnectorProperty(names = {"s3.sts_endpoint"},
supported = false,
@@ -238,8 +258,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
@Override
public Map<String, String> getBackendConfigProperties() {
- Map<String, String> backendProperties =
generateBackendS3Configuration(s3ConnectionMaximum,
- s3ConnectionRequestTimeoutS, s3ConnectionTimeoutS,
String.valueOf(usePathStyle));
+ Map<String, String> backendProperties =
generateBackendS3Configuration();
if (StringUtils.isNotBlank(s3ExternalId)
&& StringUtils.isNotBlank(s3IAMRole)) {
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
index adb59dbf249..5d718e0cbf2 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/COSPropertiesTest.java
@@ -45,10 +45,10 @@ public class COSPropertiesTest {
origProps.put("cos.access_key", "myCOSAccessKey");
origProps.put("cos.secret_key", "myCOSSecretKey");
origProps.put("cos.region", "ap-beijing-1");
- origProps.put("connection.maximum", "88");
- origProps.put("connection.request.timeout", "100");
- origProps.put("connection.timeout", "1000");
- origProps.put("use_path_style", "true");
+ origProps.put("cos.connection.maximum", "88");
+ origProps.put("cos.connection.request.timeout", "100");
+ origProps.put("cos.connection.timeout", "1000");
+ origProps.put("cos.use_path_style", "true");
origProps.put(StorageProperties.FS_COS_SUPPORT, "true");
origProps.put("test_non_storage_param", "6000");
Assertions.assertThrowsExactly(IllegalArgumentException.class, () ->
StorageProperties.createAll(origProps), "Invalid endpoint format:
https://cos.example.com";);
@@ -81,9 +81,9 @@ public class COSPropertiesTest {
origProps.put("cos.access_key", "myCOSAccessKey");
origProps.put("cos.secret_key", "myCOSSecretKey");
origProps.put("test_non_storage_param", "6000");
- origProps.put("connection.maximum", "88");
- origProps.put("connection.request.timeout", "100");
- origProps.put("connection.timeout", "1000");
+ origProps.put("cos.connection.maximum", "88");
+ origProps.put("cos.connection.request.timeout", "100");
+ origProps.put("cos.connection.timeout", "1000");
origProps.put(StorageProperties.FS_COS_SUPPORT, "true");
//origProps.put("cos.region", "ap-beijing");
@@ -106,7 +106,7 @@ public class COSPropertiesTest {
Assertions.assertEquals("100", s3Props.get("AWS_REQUEST_TIMEOUT_MS"));
Assertions.assertEquals("1000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
Assertions.assertEquals("false", s3Props.get("use_path_style"));
- origProps.put("use_path_style", "true");
+ origProps.put("cos.use_path_style", "true");
cosProperties = (COSProperties)
StorageProperties.createAll(origProps).get(0);
s3Props = cosProperties.generateBackendS3Configuration();
Assertions.assertEquals("true", s3Props.get("use_path_style"));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
index e45e5b07553..e700ede2de3 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OBSPropertyTest.java
@@ -62,10 +62,10 @@ public class OBSPropertyTest {
origProps.put("obs.access_key", "myOBSAccessKey");
origProps.put("obs.secret_key", "myOBSSecretKey");
origProps.put("obs.endpoint", "obs.cn-north-4.myhuaweicloud.com");
- origProps.put("connection.maximum", "88");
- origProps.put("connection.request.timeout", "100");
- origProps.put("connection.timeout", "1000");
- origProps.put("use_path_style", "true");
+ origProps.put("obs.connection.maximum", "88");
+ origProps.put("obs.connection.request.timeout", "100");
+ origProps.put("obs.connection.timeout", "1000");
+ origProps.put("obs.use_path_style", "true");
origProps.put("test_non_storage_param", "test_non_storage_value");
origProps.put(StorageProperties.FS_OBS_SUPPORT, "true");
OBSProperties obsProperties = (OBSProperties)
StorageProperties.createAll(origProps).get(0);
@@ -88,7 +88,7 @@ public class OBSPropertyTest {
Assertions.assertEquals("100", s3Props.get("AWS_REQUEST_TIMEOUT_MS"));
Assertions.assertEquals("1000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
Assertions.assertEquals("true", s3Props.get("use_path_style"));
- origProps.remove("use_path_style");
+ origProps.remove("obs.use_path_style");
obsProperties = (OBSProperties)
StorageProperties.createAll(origProps).get(0);
s3Props = obsProperties.getBackendConfigProperties();
Assertions.assertEquals("false", s3Props.get("use_path_style"));
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
index 0eee9b49761..20efe3ebe97 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/storage/OSSPropertiesTest.java
@@ -63,10 +63,10 @@ public class OSSPropertiesTest {
origProps.put("oss.secret_key", "myOSSSecretKey");
origProps.put("oss.endpoint", "oss-cn-beijing-internal.aliyuncs.com");
origProps.put(StorageProperties.FS_OSS_SUPPORT, "true");
- origProps.put("connection.maximum", "88");
- origProps.put("connection.request.timeout", "100");
- origProps.put("connection.timeout", "1000");
- origProps.put("use_path_style", "true");
+ origProps.put("oss.connection.maximum", "88");
+ origProps.put("oss.connection.request.timeout", "100");
+ origProps.put("oss.connection.timeout", "1000");
+ origProps.put("oss.use_path_style", "true");
origProps.put("test_non_storage_param", "6000");
OSSProperties ossProperties = (OSSProperties)
StorageProperties.createAll(origProps).get(0);
Map<String, String> s3Props;
@@ -90,7 +90,7 @@ public class OSSPropertiesTest {
Assertions.assertEquals("100", s3Props.get("AWS_REQUEST_TIMEOUT_MS"));
Assertions.assertEquals("1000",
s3Props.get("AWS_CONNECTION_TIMEOUT_MS"));
Assertions.assertEquals("true", s3Props.get("use_path_style"));
- origProps.remove("use_path_style");
+ origProps.remove("oss.use_path_style");
ossProperties = (OSSProperties)
StorageProperties.createAll(origProps).get(0);
s3Props = ossProperties.generateBackendS3Configuration();
Assertions.assertEquals("false", s3Props.get("use_path_style"));
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
