This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f30a9771e4f [chore](paimon)Unify object storage access via HDFS
protocol (#54776)
f30a9771e4f is described below
commit f30a9771e4f7081b19020869ffc82f564fc30b79
Author: Calvin Kirs <[email protected]>
AuthorDate: Fri Aug 15 18:33:39 2025 +0800
[chore](paimon)Unify object storage access via HDFS protocol (#54776)
#### Paimon protocol limitation
Paimon supports multiple storage backends but has strict protocol rules.
For example, S3 paths must be accessed using the s3 protocol, not s3a.
To ensure compatibility and reduce special handling, this PR switches to
use HDFS’s s3a protocol consistently for S3 access in Paimon.
#### COS compatibility with higher JDK versions
On higher JDK versions (e.g., JDK 17), some methods in COS’s native HDFS
implementation are no longer exposed publicly.
This PR adds the necessary extra configuration to ensure COS works with
the HDFS S3 protocol in JDK 17+ environments.
---
.../docker-compose/hive/hadoop-hive-3x.env.tpl | 2 +
.../docker-compose/kerberos/hadoop-hive.env.tpl | 2 +
.../metastore/AbstractPaimonProperties.java | 31 -------
.../metastore/HMSGlueMetaStoreProperties.java | 4 +
.../storage/AbstractS3CompatibleProperties.java | 1 +
.../datasource/property/storage/COSProperties.java | 4 +-
.../datasource/property/storage/OSSProperties.java | 43 ++++++----
.../property/metastore/PaimonCatalogTest.java | 95 ++++++++++++++++++++++
8 files changed, 135 insertions(+), 47 deletions(-)
diff --git a/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
b/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
index c8afe5046d0..4675fbf8053 100644
--- a/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
+++ b/docker/thirdparties/docker-compose/hive/hadoop-hive-3x.env.tpl
@@ -20,6 +20,7 @@
HIVE_SITE_CONF_hive_metastore_event_db_notification_api_auth=false
HIVE_SITE_CONF_hive_metastore_dml_events=true
HIVE_SITE_CONF_hive_metastore_transactional_event_listeners=org.apache.hive.hcatalog.listener.DbNotificationListener
HIVE_SITE_CONF_hive_stats_column_autogather=false
+HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk}
HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk}
@@ -33,6 +34,7 @@ HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk}
HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk}
HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion}
HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem
+HIVE_SITE_CONF_fs_cos_impl=org.apache.hadoop.fs.CosFileSystem
HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN
HIVE_SITE_CONF_fs_oss_impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk}
diff --git a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
index f06c1ef7d7a..3c453ff0fbd 100644
--- a/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
+++ b/docker/thirdparties/docker-compose/kerberos/hadoop-hive.env.tpl
@@ -64,6 +64,7 @@
YARN_CONF_yarn_resourcemanager_webapp_address=${HOST}:${YARN_RM_WEBAPP_PORT}
YARN_CONF_yarn_nodemanager_localizer_address=${HOST}:${YARN_NM_LOCAL_PORT}
YARN_CONF_yarn_nodemanager_webapp_address=${HOST}:${YARN_NM_WEBAPP_PORT}
+HIVE_SITE_CONF_fs_s3_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_impl=org.apache.hadoop.fs.s3a.S3AFileSystem
HIVE_SITE_CONF_fs_s3a_access_key=${AWSAk}
HIVE_SITE_CONF_fs_s3a_secret_key=${AWSSk}
@@ -77,6 +78,7 @@ HIVE_SITE_CONF_fs_cosn_userinfo_secretId=${COSAk}
HIVE_SITE_CONF_fs_cosn_userinfo_secretKey=${COSSk}
HIVE_SITE_CONF_fs_cosn_bucket_region=${COSRegion}
HIVE_SITE_CONF_fs_cosn_impl=org.apache.hadoop.fs.CosFileSystem
+HIVE_SITE_CONF_fs_cos_impl=org.apache.hadoop.fs.CosFileSystem
HIVE_SITE_CONF_fs_AbstractFileSystem_cosn_impl=org.apache.hadoop.fs.CosN
HIVE_SITE_CONF_fs_oss_impl=org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem
HIVE_SITE_CONF_fs_oss_accessKeyId=${OSSAk}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
index 7602758d2b0..040d04a1aad 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java
@@ -19,7 +19,6 @@ package org.apache.doris.datasource.property.metastore;
import org.apache.doris.common.security.authentication.ExecutionAuthenticator;
import org.apache.doris.datasource.property.ConnectorProperty;
-import org.apache.doris.datasource.property.storage.S3Properties;
import org.apache.doris.datasource.property.storage.StorageProperties;
import lombok.Getter;
@@ -59,34 +58,6 @@ public abstract class AbstractPaimonProperties extends
MetastoreProperties {
public abstract Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList);
- /**
- * Adapt S3 storage properties for Apache Paimon's S3 file system.
- *
- * <p>Paimon's S3 file system does not follow the standard
Hadoop-compatible
- * configuration keys (like fs.s3a.access.key). Instead, it expects
specific
- * keys such as "s3.access.key", "s3.secret.key", etc.
- *
- * <p>Therefore, we explicitly map our internal S3 configuration (usually
designed
- * for HDFS-compatible systems) to Paimon's expected format.
- *
- * <p>See: org.apache.paimon.s3.S3Loader
- *
- * @param storagePropertiesList the list of configured storage backends
- */
- protected void appendS3PropertiesIsNeeded(List<StorageProperties>
storagePropertiesList) {
-
- S3Properties s3Properties = (S3Properties)
storagePropertiesList.stream()
- .filter(storageProperties -> storageProperties.getType() ==
StorageProperties.Type.S3)
- .findFirst()
- .orElse(null);
- if (s3Properties != null) {
- catalogOptions.set("s3.access.key", s3Properties.getSecretKey());
- catalogOptions.set("s3.secret.key", s3Properties.getAccessKey());
- catalogOptions.set("s3.endpoint", s3Properties.getEndpoint());
- catalogOptions.set("s3.region", s3Properties.getRegion());
- }
- }
-
protected void appendCatalogOptions(List<StorageProperties>
storagePropertiesList) {
if (StringUtils.isNotBlank(warehouse)) {
catalogOptions.set(CatalogOptions.WAREHOUSE.key(), warehouse);
@@ -102,8 +73,6 @@ public abstract class AbstractPaimonProperties extends
MetastoreProperties {
}
}
});
-
- appendS3PropertiesIsNeeded(storagePropertiesList);
}
/**
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
index 477dba85ebf..1f31fe44dc2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
@@ -41,6 +41,8 @@ public class HMSGlueMetaStoreProperties extends
AbstractHMSProperties {
public static final int DEFAULT_MAX_RETRY = 5;
public static final String AWS_GLUE_SOCKET_TIMEOUT_KEY =
"aws.glue.socket-timeout";
public static final int DEFAULT_SOCKET_TIMEOUT =
ClientConfiguration.DEFAULT_SOCKET_TIMEOUT;
+ public static final String
AWS_CATALOG_CREDENTIALS_PROVIDER_FACTORY_CLASS_KEY =
+ "aws.catalog.credentials.provider.factory.class";
// ========== Fields ==========
private AWSGlueMetaStoreBaseProperties baseProperties;
@@ -104,6 +106,8 @@ public class HMSGlueMetaStoreProperties extends
AbstractHMSProperties {
hiveConf.set(AWS_GLUE_CONNECTION_TIMEOUT_KEY,
String.valueOf(awsGlueConnectionTimeout));
hiveConf.set(AWS_GLUE_SOCKET_TIMEOUT_KEY,
String.valueOf(awsGlueSocketTimeout));
hiveConf.set(AWS_GLUE_CATALOG_SEPARATOR_KEY, awsGlueCatalogSeparator);
+ hiveConf.set(AWS_CATALOG_CREDENTIALS_PROVIDER_FACTORY_CLASS_KEY,
+
"com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProviderFactory");
hiveConf.set("hive.metastore.type", "glue");
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
index 000bc75c529..509ebd81e87 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/AbstractS3CompatibleProperties.java
@@ -303,6 +303,7 @@ public abstract class AbstractS3CompatibleProperties
extends StorageProperties i
hadoopStorageConfig.set("fs.s3.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
hadoopStorageConfig.set("fs.s3a.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
hadoopStorageConfig.set("fs.s3a.endpoint", getEndpoint());
+ hadoopStorageConfig.set("fs.s3a.endpoint.region", getRegion());
hadoopStorageConfig.set("fs.s3a.access.key", getAccessKey());
hadoopStorageConfig.set("fs.s3a.secret.key", getSecretKey());
hadoopStorageConfig.set("fs.s3a.connection.maximum",
getMaxConnections());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
index 0e0f37e5566..f1820abb2b3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/COSProperties.java
@@ -130,8 +130,8 @@ public class COSProperties extends
AbstractS3CompatibleProperties {
@Override
public void initializeHadoopStorageConfig() {
super.initializeHadoopStorageConfig();
- hadoopStorageConfig.set("fs.cos.impl",
"org.apache.hadoop.fs.CosFileSystem");
- hadoopStorageConfig.set("fs.cosn.impl",
"org.apache.hadoop.fs.CosFileSystem");
+ hadoopStorageConfig.set("fs.cos.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
+ hadoopStorageConfig.set("fs.cosn.impl",
"org.apache.hadoop.fs.s3a.S3AFileSystem");
hadoopStorageConfig.set("fs.cosn.bucket.region", region);
hadoopStorageConfig.set("fs.cosn.userinfo.secretId", accessKey);
hadoopStorageConfig.set("fs.cosn.userinfo.secretKey", secretKey);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
index 45208e6f3d3..07d56648498 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSProperties.java
@@ -97,6 +97,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
private static final List<String> URI_KEYWORDS = Arrays.asList("uri",
"warehouse");
+ private static List<String> DLF_TYPE_KEYWORDS =
Arrays.asList("hive.metastore.type",
+ "iceberg.catalog.type", "paimon.catalog.type");
+
protected OSSProperties(Map<String, String> origProps) {
super(Type.OSS, origProps);
}
@@ -128,7 +131,9 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
if (StringUtils.isNotBlank(value)) {
return true;
}
-
+ if (isDlfMSType(origProps)) {
+ return true;
+ }
Optional<String> uriValue = origProps.entrySet().stream()
.filter(e -> URI_KEYWORDS.stream()
.anyMatch(key -> key.equalsIgnoreCase(e.getKey())))
@@ -155,22 +160,32 @@ public class OSSProperties extends
AbstractS3CompatibleProperties {
return isAliyunOss || isAmazonS3 || isDls;
}
+ private static boolean isDlfMSType(Map<String, String> params) {
+ return DLF_TYPE_KEYWORDS.stream()
+ .anyMatch(key -> params.containsKey(key) &&
StringUtils.isNotBlank(params.get(key))
+ && StringUtils.equalsIgnoreCase("dlf",
params.get(key)));
+ }
+
@Override
protected void setEndpointIfPossible() {
if (StringUtils.isBlank(this.endpoint) &&
StringUtils.isNotBlank(this.region)) {
- Optional<String> uriValueOpt = origProps.entrySet().stream()
- .filter(e -> URI_KEYWORDS.stream()
- .anyMatch(key -> key.equalsIgnoreCase(e.getKey())))
- .map(Map.Entry::getValue)
- .filter(Objects::nonNull)
- .filter(OSSProperties::isKnownObjectStorage)
- .findFirst();
- if (uriValueOpt.isPresent()) {
- String uri = uriValueOpt.get();
- // If the URI does not start with http(s), derive endpoint
from region
- // (http(s) URIs are handled by separate logic elsewhere)
- if (!uri.startsWith("http://";) && !uri.startsWith("https://";))
{
- this.endpoint = getOssEndpoint(region,
BooleanUtils.toBoolean(dlfAccessPublic));
+ if (isDlfMSType(origProps)) {
+ this.endpoint = getOssEndpoint(region,
BooleanUtils.toBoolean(dlfAccessPublic));
+ } else {
+ Optional<String> uriValueOpt = origProps.entrySet().stream()
+ .filter(e -> URI_KEYWORDS.stream()
+ .anyMatch(key ->
key.equalsIgnoreCase(e.getKey())))
+ .map(Map.Entry::getValue)
+ .filter(Objects::nonNull)
+ .filter(OSSProperties::isKnownObjectStorage)
+ .findFirst();
+ if (uriValueOpt.isPresent()) {
+ String uri = uriValueOpt.get();
+ // If the URI does not start with http(s), derive endpoint
from region
+ // (http(s) URIs are handled by separate logic elsewhere)
+ if (!uri.startsWith("http://";) &&
!uri.startsWith("https://";)) {
+ this.endpoint = getOssEndpoint(region,
BooleanUtils.toBoolean(dlfAccessPublic));
+ }
}
}
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
new file mode 100644
index 00000000000..59766ef9902
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/PaimonCatalogTest.java
@@ -0,0 +1,95 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource.property.metastore;
+
+import org.apache.doris.datasource.property.storage.StorageProperties;
+
+import org.apache.paimon.catalog.Catalog;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+@Disabled("only used for your local test")
+public class PaimonCatalogTest {
+ @Test
+ public void testNameSpace() throws Exception {
+ Map<String, String> pa = new HashMap<>();
+ pa.put("type", "paimon");
+ pa.put("paimon.catalog.type", "hms");
+ pa.put("hive.metastore.uris", "thrift://172.20.48.119:9383");
+ pa.put("warehouse", "s3a://doris/paimon_warehouse");
+ pa.put("s3.region", "ap-east-1");
+
+ // User must provide real Access Key / Secret Key to enable
initialization
+ pa.put("s3.access_key", "");
+ pa.put("s3.secret_key", "");
+ pa.put("s3.endpoint", "s3.ap-east-1.amazonaws.com");
+
+ Catalog catalog = initCatalog(pa);
+ if (catalog != null) {
+ catalog.listDatabases().forEach(System.out::println);
+ }
+ }
+
+ /**
+ * Initializes a Paimon HMS Catalog.
+ * <p>
+ * Initialization is skipped by default. Users must provide valid S3
+ * Access Key and Secret Key in the configuration map to enable it.
+ * <p>
+ * Steps:
+ * 1. Validate that credentials are provided.
+ * 2. Normalize and check metastore properties.
+ * 3. Create storage properties.
+ * 4. Initialize and return the Catalog instance.
+ *
+ * @param params A map containing the configuration parameters.
+ * @return Catalog instance if initialized, or {@code null} if skipped.
+ * @throws Exception If initialization fails.
+ */
+ private Catalog initCatalog(Map<String, String> params) throws Exception {
+ if (isDisabled(params)) {
+ System.out.println("Catalog initialization skipped: Missing valid
S3 Access Key/Secret Key.");
+ return null;
+ }
+
+ AbstractPaimonProperties metaStoreProps =
+ (AbstractPaimonProperties) MetastoreProperties.create(params);
+
+ metaStoreProps.initNormalizeAndCheckProps();
+
+ List<StorageProperties> storageProps =
StorageProperties.createAll(params);
+
+ return metaStoreProps.initializeCatalog("paimon_catalog",
storageProps);
+ }
+
+ /**
+ * Checks if initialization should be skipped due to missing credentials.
+ *
+ * @param params The configuration parameters.
+ * @return {@code true} if missing AK/SK, {@code false} otherwise.
+ */
+ private boolean isDisabled(Map<String, String> params) {
+ String ak = params.get("s3.access_key");
+ String sk = params.get("s3.secret_key");
+ return ak == null || ak.isEmpty() || sk == null || sk.isEmpty();
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
