This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch branch-4.2
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-4.2 by this push:
new 62cb1df3a7f branch-4.2: [improvement](s3) Default scheme-less S3
endpoints to HTTPS (#65379) (#69151)
62cb1df3a7f is described below
commit 62cb1df3a7f8124ad338e8788e4faaf71f030b07
Author: Xin Liao <[email protected]>
AuthorDate: Tue Oct 20 10:14:53 2026 +0800
branch-4.2: [improvement](s3) Default scheme-less S3 endpoints to HTTPS
(#65379) (#69151)
Pick apache/doris#65379 to branch-4.2.
Branch adaptation: retain the 4.2 S3 property and container credential
implementations. Update the 4.2 FE S3 client call sites so scheme-less
endpoints reach the shared HTTPS default. Keep persisted S3 resource
endpoints as supplied and normalize a copy for connectivity checks.
SSRF surface: the existing FE catalog connectivity test can contact an
S3 host supplied by a user with CREATE privilege on the catalog. This
change lets a scheme-less endpoint use HTTPS for that request; it adds
no new host source or request site.
Validation: `build-support/clang-format.sh`,
`build-support/check-format.sh`, and `git diff --check` passed.
`DORIS_GCC_HOME=/usr ./run-fe-ut.sh --run
org.apache.doris.common.util.S3UtilTest,org.apache.doris.catalog.S3ResourceTest`
was attempted but stopped during generated-code build because
`thirdparty/installed/bin/protoc` is missing from this worktree. BE,
Cloud, and regression tests were not run.
---------
Co-authored-by: Refrain <[email protected]>
---
be/src/common/config.cpp | 4 +-
be/src/common/config.h | 2 +-
be/src/util/s3_util.cpp | 4 +-
be/test/io/s3_client_factory_test.cpp | 29 +++++++++++
cloud/src/common/config.h | 4 +-
cloud/src/recycler/s3_accessor.cpp | 4 +-
cloud/test/util_test.cpp | 30 ++++++++++++
common/cpp/aws_common.cpp | 10 ++++
common/cpp/aws_common.h | 10 +++-
.../main/java/org/apache/doris/common/Config.java | 5 ++
.../java/org/apache/doris/catalog/S3Resource.java | 29 ++++++-----
.../java/org/apache/doris/common/util/S3Util.java | 22 ++++++---
.../AbstractS3CompatibleConnectivityTester.java | 2 +-
.../doris/datasource/iceberg/dlf/DLFCatalog.java | 7 +--
.../java/org/apache/doris/fs/obj/S3ObjStorage.java | 8 +---
.../org/apache/doris/catalog/S3ResourceTest.java | 18 +++++--
.../org/apache/doris/common/util/S3UtilTest.java | 56 ++++++++++++++++++++++
.../pipeline/vault_p0/conf/be_custom.conf | 1 +
.../pipeline/vault_p0/conf/fe_custom.conf | 1 +
.../pipeline/vault_p0/conf/recycler_custom.conf | 1 +
.../cold_heat_separation/policy/alter.groovy | 2 +-
21 files changed, 200 insertions(+), 49 deletions(-)
diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp
index bd9eff1e435..705fcdd35db 100644
--- a/be/src/common/config.cpp
+++ b/be/src/common/config.cpp
@@ -1805,8 +1805,8 @@ DEFINE_mInt32(thrift_client_open_num_tries, "1");
DEFINE_Bool(enable_index_compaction, "false");
-// http scheme in S3Client to use. E.g. http or https
-DEFINE_String(s3_client_http_scheme, "http");
+// Default HTTP scheme used by S3Client when the endpoint has no scheme.
+DEFINE_String(s3_client_http_scheme, "https");
DEFINE_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
return config == "http" || config == "https";
});
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 70c6519efef..5c7ff731260 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -1799,7 +1799,7 @@ DECLARE_mInt64(paimon_jni_writer_memory_pool_limit_bytes);
// Retry the Open num_retries time waiting 100 milliseconds between retries.
DECLARE_mInt32(thrift_client_open_num_tries);
-// http scheme in S3Client to use. E.g. http or https
+// Default HTTP scheme used by S3Client when the endpoint has no scheme.
DECLARE_String(s3_client_http_scheme);
DECLARE_mBool(ignore_schema_change_check);
diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp
index e3d2ac9be2b..0cae6f74cc2 100644
--- a/be/src/util/s3_util.cpp
+++ b/be/src/util/s3_util.cpp
@@ -444,9 +444,7 @@ std::shared_ptr<io::ObjStorageClient>
S3ClientFactory::_create_s3_client(
aws_config.connectTimeoutMs = s3_conf.connect_timeout_ms;
}
- if (config::s3_client_http_scheme == "http") {
- aws_config.scheme = Aws::Http::Scheme::HTTP;
- }
+ set_s3_client_default_http_scheme(aws_config,
config::s3_client_http_scheme);
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
config::max_s3_client_retry /*scaleFactor = 25*/,
/*retry_slow_down=*/true);
diff --git a/be/test/io/s3_client_factory_test.cpp
b/be/test/io/s3_client_factory_test.cpp
index 0882fd7847c..fc6413a0f04 100644
--- a/be/test/io/s3_client_factory_test.cpp
+++ b/be/test/io/s3_client_factory_test.cpp
@@ -18,6 +18,7 @@
#include <aws/core/auth/AWSCredentialsProviderChain.h>
#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
#include <aws/core/auth/STSCredentialsProvider.h>
+#include <aws/core/client/ClientConfiguration.h>
#include <aws/identity-management/auth/STSAssumeRoleCredentialsProvider.h>
#include <gtest/gtest.h>
@@ -29,6 +30,7 @@
#include <vector>
#include "cloud/config.h"
+#include "cpp/aws_common.h"
#include "cpp/custom_aws_credentials_provider_chain.h"
#include "io/fs/rate_limited_obj_storage_client.h"
#include "io/fs/s3_obj_storage_client.h"
@@ -236,6 +238,32 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) {
config::aws_credentials_provider_version = "v2";
}
+TEST_F(S3ClientFactoryTest, SetS3ClientDefaultHttpScheme) {
+ S3ClientFactory::instance();
+ Aws::Client::ClientConfiguration client_config;
+ client_config.endpointOverride = "example.com:9000";
+
+ set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+
+ client_config.endpointOverride = "http://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTP;
+ set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "http://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ client_config.endpointOverride = "https://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTPS;
+ set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "https://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+}
+
TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRoleArnProviderType) {
std::map<std::string, std::string> properties {
{"AWS_ENDPOINT", "s3.us-west-2.amazonaws.com"},
@@ -248,6 +276,7 @@ TEST_F(S3ClientFactoryTest,
ConvertPropertiesToS3ConfRoleArnProviderType) {
S3Conf s3_conf;
ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties,
s3_uri, &s3_conf).ok());
+ ASSERT_EQ(s3_conf.client_conf.endpoint, properties.at("AWS_ENDPOINT"));
ASSERT_EQ(s3_conf.client_conf.cred_provider_type,
CredProviderType::Default);
properties["AWS_CREDENTIALS_PROVIDER_TYPE"] = "WEB_IDENTITY";
diff --git a/cloud/src/common/config.h b/cloud/src/common/config.h
index e6558756465..f851b4303d3 100644
--- a/cloud/src/common/config.h
+++ b/cloud/src/common/config.h
@@ -322,8 +322,8 @@ CONF_String(priority_networks, "");
CONF_Bool(enable_cluster_name_check, "false");
-// http scheme in S3Client to use. E.g. http or https
-CONF_String(s3_client_http_scheme, "http");
+// Default HTTP scheme used by S3Client when the endpoint has no scheme.
+CONF_String(s3_client_http_scheme, "https");
CONF_Validator(s3_client_http_scheme, [](const std::string& config) -> bool {
return config == "http" || config == "https";
});
diff --git a/cloud/src/recycler/s3_accessor.cpp
b/cloud/src/recycler/s3_accessor.cpp
index 08effdf17fc..e5fb445a45d 100644
--- a/cloud/src/recycler/s3_accessor.cpp
+++ b/cloud/src/recycler/s3_accessor.cpp
@@ -426,9 +426,7 @@ int S3Accessor::init() {
config::instance_recycler_worker_pool_size),
(long)aws_config.maxConnections);
- if (config::s3_client_http_scheme == "http") {
- aws_config.scheme = Aws::Http::Scheme::HTTP;
- }
+ set_s3_client_default_http_scheme(aws_config,
config::s3_client_http_scheme);
// Recycler should fail fast on S3 SlowDown instead of retrying and
blocking worker threads.
aws_config.retryStrategy = std::make_shared<S3CustomRetryStrategy>(
config::max_s3_client_retry, /*retry_slow_down=*/false);
diff --git a/cloud/test/util_test.cpp b/cloud/test/util_test.cpp
index 0b20ae08171..abe91c65199 100644
--- a/cloud/test/util_test.cpp
+++ b/cloud/test/util_test.cpp
@@ -17,8 +17,10 @@
#include "cpp/util.h"
+#include <aws/core/client/ClientConfiguration.h>
#include <bthread/countdown_event.h>
+#include <chrono>
#include <stdexcept>
#include <string>
#include <string_view>
@@ -33,6 +35,7 @@
#include "cpp/sync_point.h"
#include "gtest/gtest.h"
#include "recycler/recycler.h"
+#include "recycler/s3_accessor.h"
#include "recycler/sync_executor.h"
using namespace doris::cloud;
@@ -456,3 +459,30 @@ TEST(UtilTest, test_long_normalize_http_uri) {
EXPECT_EQ(doris::normalize_http_uri(longPath), expected);
}
+
+TEST(UtilTest, test_set_s3_client_default_http_scheme) {
+ doris::cloud::S3Environment::getInstance();
+ Aws::Client::ClientConfiguration client_config =
+ doris::cloud::S3Environment::getClientConfiguration();
+
+ client_config.endpointOverride = "example.com:9000";
+ doris::set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ doris::set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "example.com:9000");
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+
+ client_config.endpointOverride = "http://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTP;
+ doris::set_s3_client_default_http_scheme(client_config, "https");
+ EXPECT_EQ(client_config.endpointOverride, "http://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTP);
+
+ client_config.endpointOverride = "https://example.com:9000";;
+ client_config.scheme = Aws::Http::Scheme::HTTPS;
+ doris::set_s3_client_default_http_scheme(client_config, "http");
+ EXPECT_EQ(client_config.endpointOverride, "https://example.com:9000";);
+ EXPECT_EQ(client_config.scheme, Aws::Http::Scheme::HTTPS);
+}
diff --git a/common/cpp/aws_common.cpp b/common/cpp/aws_common.cpp
index 885f7e7421e..0f2d900d8cd 100644
--- a/common/cpp/aws_common.cpp
+++ b/common/cpp/aws_common.cpp
@@ -18,6 +18,7 @@
#include "aws_common.h"
#include <aws/core/auth/GeneralHTTPCredentialsProvider.h>
+#include <aws/core/client/ClientConfiguration.h>
#include <aws/core/platform/Environment.h>
#include <aws/core/utils/memory/AWSMemory.h>
#include <glog/logging.h>
@@ -142,4 +143,13 @@ std::string get_valid_ca_cert_path(const
std::vector<std::string>& ca_cert_file_
}
return "";
}
+
+void set_s3_client_default_http_scheme(Aws::Client::ClientConfiguration&
client_config,
+ const std::string& scheme) {
+ if (client_config.endpointOverride.starts_with("http://";) ||
+ client_config.endpointOverride.starts_with("https://";)) {
+ return;
+ }
+ client_config.scheme = scheme == "http" ? Aws::Http::Scheme::HTTP :
Aws::Http::Scheme::HTTPS;
+}
}
diff --git a/common/cpp/aws_common.h b/common/cpp/aws_common.h
index 5caff27bec1..275d35889b6 100644
--- a/common/cpp/aws_common.h
+++ b/common/cpp/aws_common.h
@@ -26,6 +26,10 @@ namespace Aws::Auth {
class AWSCredentialsProvider;
}
+namespace Aws::Client {
+struct ClientConfiguration;
+}
+
namespace doris {
//AWS Credentials Provider Type
enum class CredProviderType {
@@ -72,4 +76,8 @@ namespace doris {
std::string get_valid_ca_cert_path(const std::vector<std::string>&
ca_cert_file_paths);
- } // namespace doris
\ No newline at end of file
+ // Configures the default S3 client transport scheme for endpoints without
an explicit scheme.
+ void set_s3_client_default_http_scheme(Aws::Client::ClientConfiguration&
client_config,
+ const std::string& scheme);
+
+ } // namespace doris
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
index 95752e6c3ed..c311f4c55ec 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -3688,6 +3688,11 @@ public class Config extends ConfigBase {
+ "it is intentionally not modifiable at runtime via ADMIN
SET FRONTEND CONFIG."})
public static String[] s3_load_endpoint_white_list = {};
+ @ConfField(description = {
+ "The default scheme for S3 endpoints without an explicit scheme.
Valid values are http and https."},
+ options = {"http", "https"})
+ public static String s3_client_http_scheme = "https";
+
@ConfField(mutable = true, description = {
"对于确定性的 S3 路径（无通配符如 *, ?），使用 HEAD 请求代替 ListObjects 来避免需要
ListBucket 权限。"
+ "花括号模式 {1,2,3} 和非否定方括号模式 [abc] 会展开为具体路径。"
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
index 4e4d5b7b63b..948bfe470cc 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.proc.BaseProcResult;
import org.apache.doris.common.util.PrintableMap;
+import org.apache.doris.common.util.S3Util;
import
org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties;
import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
@@ -101,20 +102,19 @@ public class S3Resource extends Resource {
LOG.debug("s3 info need check validity : {}", needCheck);
}
- // the endpoint for ping need add uri scheme.
- String pingEndpoint = properties.get(S3Properties.ENDPOINT);
- if (!pingEndpoint.startsWith("http://";) &&
!pingEndpoint.startsWith("https://";)) {
- pingEndpoint = "http://"; + properties.get(S3Properties.ENDPOINT);
- properties.put(S3Properties.ENDPOINT, pingEndpoint);
- properties.put(S3Properties.Env.ENDPOINT, pingEndpoint);
- }
- String region = S3Properties.getRegionOfEndpoint(pingEndpoint);
+ String endpoint = properties.get(S3Properties.ENDPOINT);
+ properties.put(S3Properties.Env.ENDPOINT, endpoint);
+ String region = S3Properties.getRegionOfEndpoint(endpoint);
properties.putIfAbsent(S3Properties.REGION, region);
if (needCheck) {
+ Map<String, String> pingProperties = new HashMap<>(properties);
+ String pingEndpoint = S3Util.buildEndpointUrl(endpoint);
+ pingProperties.put(S3Properties.ENDPOINT, pingEndpoint);
+ pingProperties.put(S3Properties.Env.ENDPOINT, pingEndpoint);
String bucketName = properties.get(S3Properties.BUCKET);
String rootPath = properties.get(S3Properties.ROOT_PATH);
- pingS3(bucketName, rootPath, properties);
+ pingS3(bucketName, rootPath, pingProperties);
}
// optional
S3Properties.optionalS3Property(properties);
@@ -188,6 +188,9 @@ public class S3Resource extends Resource {
}
// compatible with old version, Need convert if modified properties
map uses old properties.
S3Properties.convertToStdProperties(properties);
+ if (!Strings.isNullOrEmpty(properties.get(S3Properties.ENDPOINT))) {
+ properties.put(S3Properties.Env.ENDPOINT,
properties.get(S3Properties.ENDPOINT));
+ }
boolean needCheck = isNeedCheck(properties);
if (LOG.isDebugEnabled()) {
LOG.debug("s3 info need check validity : {}", needCheck);
@@ -196,6 +199,9 @@ public class S3Resource extends Resource {
S3Properties.requiredS3PingProperties(this.properties);
Map<String, String> changedProperties = new
HashMap<>(this.properties);
changedProperties.putAll(properties);
+ String endpoint =
S3Util.buildEndpointUrl(changedProperties.get(S3Properties.ENDPOINT));
+ changedProperties.put(S3Properties.ENDPOINT, endpoint);
+ changedProperties.put(S3Properties.Env.ENDPOINT, endpoint);
String bucketName = properties.getOrDefault(S3Properties.BUCKET,
this.properties.get(S3Properties.BUCKET));
String rootPath = properties.getOrDefault(S3Properties.ROOT_PATH,
this.properties.get(S3Properties.ROOT_PATH));
@@ -240,8 +246,8 @@ public class S3Resource extends Resource {
String token = properties.getOrDefault(S3Properties.SESSION_TOKEN,
this.properties.get(S3Properties.SESSION_TOKEN));
String endpoint = properties.getOrDefault(S3Properties.ENDPOINT,
this.properties.get(S3Properties.ENDPOINT));
- String pingEndpoint = "http://"; + endpoint;
- String region = S3Properties.getRegionOfEndpoint(pingEndpoint);
+ String pingEndpoint = S3Util.buildEndpointUrl(endpoint);
+ String region = S3Properties.getRegionOfEndpoint(endpoint);
properties.putIfAbsent(S3Properties.REGION, region);
return new CloudCredentialWithEndpoint(pingEndpoint, region, ak, sk,
token);
}
@@ -284,4 +290,3 @@ public class S3Resource extends Resource {
readUnlock();
}
}
-
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
index 7b2d213697d..be75e961fde 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/S3Util.java
@@ -94,7 +94,7 @@ public class S3Util {
}
@Deprecated
- public static S3Client buildS3Client(URI endpoint, String region,
CloudCredential credential,
+ public static S3Client buildS3Client(String endpoint, String region,
CloudCredential credential,
boolean isUsePathStyle) {
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
.builder()
@@ -117,7 +117,7 @@ public class S3Util {
return S3Client.builder()
.httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30))
.connectionTimeout(Duration.ofSeconds(30)).build())
- .endpointOverride(endpoint)
+ .endpointOverride(URI.create(buildEndpointUrl(endpoint)))
.credentialsProvider(getAwsCredencialsProvider(credential))
.region(Region.of(region))
.overrideConfiguration(clientConf)
@@ -220,7 +220,7 @@ public class S3Util {
sessionToken, roleArn, externalId);
}
- public static S3Client buildS3Client(URI endpoint, String region, boolean
isUsePathStyle,
+ public static S3Client buildS3Client(String endpoint, String region,
boolean isUsePathStyle,
AwsCredentialsProvider credential) {
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
.builder()
@@ -243,7 +243,7 @@ public class S3Util {
return S3Client.builder()
.httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30))
.connectionTimeout(Duration.ofSeconds(30)).build())
- .endpointOverride(endpoint)
+ .endpointOverride(URI.create(buildEndpointUrl(endpoint)))
.credentialsProvider(credential)
.region(Region.of(region))
.overrideConfiguration(clientConf)
@@ -255,7 +255,7 @@ public class S3Util {
.build();
}
- public static S3Client buildS3Client(URI endpoint, String region, boolean
isUsePathStyle, String accessKey,
+ public static S3Client buildS3Client(String endpoint, String region,
boolean isUsePathStyle, String accessKey,
String secretKey, String sessionToken, String roleArn, String
externalId) {
EqualJitterBackoffStrategy backoffStrategy = EqualJitterBackoffStrategy
.builder()
@@ -275,11 +275,12 @@ public class S3Util {
// using AwsS3V4Signer
.putAdvancedOption(SdkAdvancedClientOption.SIGNER,
AwsS3V4Signer.create())
.build();
+ URI endpointUri = URI.create(buildEndpointUrl(endpoint));
return S3Client.builder()
.httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30))
.connectionTimeout(Duration.ofSeconds(30)).build())
- .endpointOverride(endpoint)
- .credentialsProvider(getAwsCredencialsProvider(endpoint,
region, accessKey, secretKey,
+ .endpointOverride(endpointUri)
+ .credentialsProvider(getAwsCredencialsProvider(endpointUri,
region, accessKey, secretKey,
sessionToken, roleArn, externalId))
.region(Region.of(region))
.overrideConfiguration(clientConf)
@@ -697,6 +698,13 @@ public class S3Util {
}
}
+ public static String buildEndpointUrl(String endpoint) {
+ if (endpoint.startsWith("http://";) || endpoint.startsWith("https://";))
{
+ return endpoint;
+ }
+ return Config.s3_client_http_scheme + "://" + endpoint;
+ }
+
/**
* Check if a path pattern is deterministic, meaning all file paths can be
determined
* without listing. A pattern is deterministic if it contains no true
wildcard characters
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
index 551448dc767..25b0e5e2995 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/connectivity/AbstractS3CompatibleConnectivityTester.java
@@ -62,7 +62,7 @@ public abstract class AbstractS3CompatibleConnectivityTester
implements StorageC
String endpoint = properties.getEndpoint();
try (S3Client client = S3Util.buildS3Client(
- URI.create(endpoint),
+ endpoint,
properties.getRegion(),
Boolean.parseBoolean(properties.getUsePathStyle()),
properties.getAwsCredentialsProvider())) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
index fb90260f09a..d5dc5795315 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/dlf/DLFCatalog.java
@@ -30,7 +30,6 @@ import org.apache.iceberg.aws.s3.S3FileIO;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.io.FileIO;
-import java.net.URI;
import java.util.Map;
public class DLFCatalog extends HiveCompatibleCatalog {
@@ -61,11 +60,7 @@ public class DLFCatalog extends HiveCompatibleCatalog {
boolean isUsePathStyle =
Boolean.parseBoolean(ossProperties.getUsePathStyle());
// s3 file io just supports s3-like endpoint
String s3Endpoint = endpoint.replace("oss-" + region, "s3.oss-" +
region);
- if (!s3Endpoint.contains("://")) {
- s3Endpoint = "http://"; + s3Endpoint;
- }
- URI endpointUri = URI.create(s3Endpoint);
- FileIO io = new S3FileIO(() -> S3Util.buildS3Client(endpointUri,
region, credential, isUsePathStyle));
+ FileIO io = new S3FileIO(() -> S3Util.buildS3Client(s3Endpoint,
region, credential, isUsePathStyle));
io.initialize(properties);
return io;
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
index 5f14506c4de..de342349022 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java
@@ -83,7 +83,6 @@ import software.amazon.awssdk.services.sts.model.Credentials;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.InputStream;
-import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
@@ -116,12 +115,7 @@ public class S3ObjStorage implements ObjStorage<S3Client> {
@Override
public S3Client getClient() {
if (client == null) {
- String endpointStr = s3Properties.getEndpoint();
- if (!endpointStr.contains("://")) {
- endpointStr = "http://"; + endpointStr;
- }
- URI endpoint = URI.create(endpointStr);
- client = S3Util.buildS3Client(endpoint, s3Properties.getRegion(),
+ client = S3Util.buildS3Client(s3Properties.getEndpoint(),
s3Properties.getRegion(),
isUsePathStyle, s3Properties.getAwsCredentialsProvider());
}
return client;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
index 8a8d23d2a64..e725b967315 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java
@@ -196,7 +196,8 @@ public class S3ResourceTest {
Assert.assertEquals("s3_1", rS3Resource1.getName());
Assert.assertEquals("s3_2", rS3Resource2.getName());
- Assert.assertEquals(rS3Resource2.getProperty(S3Properties.ENDPOINT),
"http://aaa";);
+ Assert.assertEquals(rS3Resource2.getProperty(S3Properties.ENDPOINT),
"aaa");
+
Assert.assertEquals(rS3Resource2.getProperty(S3Properties.Env.ENDPOINT), "aaa");
Assert.assertEquals(rS3Resource2.getProperty(S3Properties.REGION),
"bbb");
Assert.assertEquals(rS3Resource2.getProperty(S3Properties.ROOT_PATH),
"/path/to/root");
Assert.assertEquals(rS3Resource2.getProperty(S3Properties.ACCESS_KEY),
"xxx");
@@ -228,11 +229,22 @@ public class S3ResourceTest {
Map<String, String> modify = new HashMap<>();
modify.put("s3.access_key", "aaa");
s3Resource.modifyProperties(modify);
+
+ modify.clear();
+ modify.put(S3Properties.ENDPOINT, "new-endpoint");
+ s3Resource.modifyProperties(modify);
+ Assert.assertEquals("new-endpoint",
s3Resource.getProperty(S3Properties.ENDPOINT));
+ Assert.assertEquals("new-endpoint",
s3Resource.getProperty(S3Properties.Env.ENDPOINT));
+
+ modify.clear();
+ modify.put(S3Properties.Env.ENDPOINT, "http://other-endpoint";);
+ s3Resource.modifyProperties(modify);
+ Assert.assertEquals("http://other-endpoint";,
s3Resource.getProperty(S3Properties.ENDPOINT));
+ Assert.assertEquals("http://other-endpoint";,
s3Resource.getProperty(S3Properties.Env.ENDPOINT));
}
@Test
- public void testHttpScheme() throws DdlException {
- // if https:// is set, it should be replaced with http://
+ public void testExplicitSchemeIsPreserved() throws DdlException {
ImmutableMap<String, String> properties = ImmutableMap.of(
"AWS_ENDPOINT", "https://aaa";,
"AWS_REGION", "bbb",
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
index ed3cdd56328..764196ae8b8 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/util/S3UtilTest.java
@@ -17,9 +17,17 @@
package org.apache.doris.common.util;
+import org.apache.doris.common.Config;
+
+import org.junit.After;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.services.s3.S3Client;
+import java.net.URI;
import java.nio.file.FileSystems;
import java.nio.file.PathMatcher;
import java.nio.file.Paths;
@@ -27,6 +35,54 @@ import java.util.Arrays;
import java.util.List;
public class S3UtilTest {
+ private String originalS3ClientHttpScheme;
+
+ @Before
+ public void setUp() {
+ originalS3ClientHttpScheme = Config.s3_client_http_scheme;
+ }
+
+ @After
+ public void tearDown() {
+ Config.s3_client_http_scheme = originalS3ClientHttpScheme;
+ }
+
+ @Test
+ public void testBuildEndpointUrlDefaultsToHttps() {
+ Config.s3_client_http_scheme = "https";
+ Assert.assertEquals("https://s3.us-east-1.amazonaws.com";,
+ S3Util.buildEndpointUrl("s3.us-east-1.amazonaws.com"));
+ }
+
+ @Test
+ public void testBuildEndpointUrlUsesConfiguredHttpScheme() {
+ Config.s3_client_http_scheme = "http";
+ Assert.assertEquals("http://127.0.0.1:9000";,
+ S3Util.buildEndpointUrl("127.0.0.1:9000"));
+ }
+
+ @Test
+ public void testBuildEndpointUrlPreservesExplicitSchemes() {
+ Config.s3_client_http_scheme = "https";
+ Assert.assertEquals("http://127.0.0.1:9000";,
+ S3Util.buildEndpointUrl("http://127.0.0.1:9000";));
+
+ Config.s3_client_http_scheme = "http";
+ Assert.assertEquals("https://s3.us-east-1.amazonaws.com";,
+ S3Util.buildEndpointUrl("https://s3.us-east-1.amazonaws.com";));
+ }
+
+ @Test
+ public void testBuildS3ClientAppliesDefaultSchemeAtClientCreation() {
+ Config.s3_client_http_scheme = "https";
+ try (S3Client client = S3Util.buildS3Client(
+ "127.0.0.1:9000", "us-east-1", true,
+
StaticCredentialsProvider.create(AwsBasicCredentials.create("ak", "sk")))) {
+ Assert.assertEquals(URI.create("https://127.0.0.1:9000";),
+
client.serviceClientConfiguration().endpointOverride().get());
+ }
+ }
+
@Test
public void testExtendGlobNumberRange_simpleRange() {
diff --git a/regression-test/pipeline/vault_p0/conf/be_custom.conf
b/regression-test/pipeline/vault_p0/conf/be_custom.conf
index 779eb5a77a5..3a2a0e16213 100644
--- a/regression-test/pipeline/vault_p0/conf/be_custom.conf
+++ b/regression-test/pipeline/vault_p0/conf/be_custom.conf
@@ -11,6 +11,7 @@ enable_file_cache=true
enable_file_cache_query_limit=true
file_cache_max_file_segment_size=1048576
s3_write_buffer_whole_size=52428800
+s3_client_http_scheme = http
enable_vertical_compaction=true
fuzzy_vertical_compaction=true
vacuum_stale_rowsets_interval_seconds=60
diff --git a/regression-test/pipeline/vault_p0/conf/fe_custom.conf
b/regression-test/pipeline/vault_p0/conf/fe_custom.conf
index f62ffa19fc7..9b622795168 100644
--- a/regression-test/pipeline/vault_p0/conf/fe_custom.conf
+++ b/regression-test/pipeline/vault_p0/conf/fe_custom.conf
@@ -12,6 +12,7 @@ enable_mtmv = true
remote_fragment_exec_timeout_ms=60000
dynamic_partition_check_interval_seconds=10
use_fuzzy_session_variable=true
+s3_client_http_scheme = http
enable_cloud_snapshot_version = true
enable_auto_collect_statistics = false
diff --git a/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
b/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
index 62deff2b870..3c3dc627122 100644
--- a/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
+++ b/regression-test/pipeline/vault_p0/conf/recycler_custom.conf
@@ -1,2 +1,3 @@
# below lines will be appended to the default doris_cloud.conf when deploying
recycler
brpc_listen_port = 6000
+s3_client_http_scheme = http
diff --git a/regression-test/suites/cold_heat_separation/policy/alter.groovy
b/regression-test/suites/cold_heat_separation/policy/alter.groovy
index cfd8e0da6fc..22903e672ba 100644
--- a/regression-test/suites/cold_heat_separation/policy/alter.groovy
+++ b/regression-test/suites/cold_heat_separation/policy/alter.groovy
@@ -174,7 +174,7 @@ suite("alter_policy") {
// AWS_CONNECTION_TIMEOUT_MS
assertEquals(show_alter_result[5][3], "2222")
// AWS_ENDPOINT
- assertEquals(show_alter_result[6][3], "http://bj.s3.comaaaa";)
+ assertEquals(show_alter_result[6][3], "bj.s3.comaaaa")
// AWS_REGION
assertEquals(show_alter_result[7][3], "bj")
// s3_rootpath
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
