This is an automated email from the ASF dual-hosted git repository.
kirs pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new df3b1f26fcc [Fix](refactor-params)Fix DLF Access (#53210)
df3b1f26fcc is described below
commit df3b1f26fcc26246d6a46ae4d103827d23ca1f2d
Author: Calvin Kirs <[email protected]>
AuthorDate: Tue Jul 15 09:41:07 2025 +0800
[Fix](refactor-params)Fix DLF Access (#53210)
### What problem does this PR solve?
#50238
[](https://github.com/apache/doris/commit/97402e3956949b00073527254a0640c337be0625)
####
When integrating Alibaba Cloud DLF (Data Lake Formation) as a metadata
source, we encountered the following issues:
catalogId was not explicitly specified, which caused DLF to return only
a partial list of databases or tables.
When using a DLF-provided endpoint, it must be converted to an
OSSHDFS-compatible endpoint (e.g., oss:// or oss+hdfs://) in order to
support subsequent data access or file operations.
---
.../property/metastore/AliyunDLFBaseProperties.java | 11 +++++++++++
.../metastore/HMSAliyunDLFMetaStoreProperties.java | 17 +++++++++++------
.../datasource/property/storage/OSSHdfsProperties.java | 11 ++++++++++-
.../doris/datasource/property/storage/S3Properties.java | 7 +++++++
4 files changed, 39 insertions(+), 7 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
index 8ab305aa861..0744a716b18 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
@@ -20,6 +20,8 @@ package org.apache.doris.datasource.property.metastore;
import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
import org.apache.doris.datasource.property.ConnectorProperty;
+import com.aliyun.datalake.metastore.common.DataLakeConfig;
+
import java.util.Map;
public class AliyunDLFBaseProperties {
@@ -49,11 +51,20 @@ public class AliyunDLFBaseProperties {
description = "The uid of the Aliyun DLF.")
protected String dlfUid = "";
+ @ConnectorProperty(names = {"dlf.catalog.id"},
+ description = "The catalog id of the Aliyun DLF. If not set, it
will be the same as dlf.uid.")
+ protected String dlfCatalogId = "";
+
@ConnectorProperty(names = {"dlf.access.public",
"dlf.catalog.accessPublic"},
required = false,
description = "Enable public access to Aliyun DLF.")
protected String dlfAccessPublic = "false";
+ @ConnectorProperty(names = {DataLakeConfig.CATALOG_PROXY_MODE},
+ required = false,
+ description = "The proxy mode of the Aliyun DLF. Default is
DLF_ONLY.")
+ protected String dlfProxyMode = "DLF_ONLY";
+
public static AliyunDLFBaseProperties of(Map<String, String> properties) {
AliyunDLFBaseProperties propertiesObj = new AliyunDLFBaseProperties();
ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj,
properties);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStoreProperties.java
index e971f0fe58b..2a7dc6a12d2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSAliyunDLFMetaStoreProperties.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.property.metastore;
+import com.aliyun.datalake.metastore.common.DataLakeConfig;
import org.apache.hadoop.hive.conf.HiveConf;
import java.util.Map;
@@ -40,12 +41,16 @@ public class HMSAliyunDLFMetaStoreProperties extends
AbstractHMSProperties {
// @see com.aliyun.datalake.metastore.hive.common.utils.ConfigUtils
// todo support other parameters
hiveConf = new HiveConf();
- hiveConf.set("dlf.catalog.accessKeyId", baseProperties.dlfAccessKey);
- hiveConf.set("dlf.catalog.accessKeySecret",
baseProperties.dlfSecretKey);
- hiveConf.set("dlf.catalog.endpoint", baseProperties.dlfEndpoint);
- hiveConf.set("dlf.catalog.region", baseProperties.dlfRegion);
- hiveConf.set("dlf.catalog.securityToken",
baseProperties.dlfSessionToken);
- hiveConf.set("dlf.catalog.id", baseProperties.dlfUid);
+ hiveConf.set(DataLakeConfig.CATALOG_ACCESS_KEY_ID,
baseProperties.dlfAccessKey);
+ hiveConf.set(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET,
baseProperties.dlfSecretKey);
+ hiveConf.set(DataLakeConfig.CATALOG_ENDPOINT,
baseProperties.dlfEndpoint);
+ hiveConf.set(DataLakeConfig.CATALOG_REGION_ID,
baseProperties.dlfRegion);
+ hiveConf.set(DataLakeConfig.CATALOG_SECURITY_TOKEN,
baseProperties.dlfSessionToken);
+ hiveConf.set(DataLakeConfig.CATALOG_USER_ID, baseProperties.dlfUid);
+ hiveConf.set(DataLakeConfig.CATALOG_ID, baseProperties.dlfCatalogId);
+ hiveConf.set(DataLakeConfig.CATALOG_PROXY_MODE,
baseProperties.dlfProxyMode);
+ hiveConf.set("hive.metastore.type", "dlf");
+ hiveConf.set("type", "hms");
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
index 8aeb6c307e6..8c29addaf57 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/OSSHdfsProperties.java
@@ -58,7 +58,8 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
description = "The access key of OSS.")
protected String accessKey = "";
- @ConnectorProperty(names = {"oss.hdfs.secret_key", "oss.secret_key"},
description = "The secret key of OSS.")
+ @ConnectorProperty(names = {"oss.hdfs.secret_key", "oss.secret_key",
"dlf.secret_key", "dlf.catalog.secret_key"},
+ description = "The secret key of OSS.")
protected String secretKey = "";
@ConnectorProperty(names = {"oss.hdfs.region", "oss.region", "dlf.region"},
@@ -125,6 +126,13 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
}
}
+ private void convertDlfToOssEndpointIfNeeded() {
+ if (this.endpoint.contains("dlf")) {
+ // If the endpoint already contains "oss-dls.aliyuncs.com", return
it as is.
+ this.endpoint = this.region + ".oss-dls.aliyuncs.com";
+ }
+ }
+
public static Optional<String> extractRegion(String endpoint) {
for (Pattern pattern : ENDPOINT_PATTERN) {
Matcher matcher = pattern.matcher(endpoint.toLowerCase());
@@ -160,6 +168,7 @@ public class OSSHdfsProperties extends
HdfsCompatibleProperties {
}
this.region = regionOptional.get();
}
+ convertDlfToOssEndpointIfNeeded();
if (StringUtils.isBlank(fsDefaultFS)) {
this.fsDefaultFS =
HdfsPropertiesUtils.extractDefaultFsFromUri(origProps, supportSchema);
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index 5953ffef8b8..f036494875e 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -157,6 +157,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
@Override
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
+ convertGlueToS3EndpointIfNeeded();
if (StringUtils.isNotBlank(accessKey) &&
StringUtils.isNotBlank(secretKey)) {
return;
}
@@ -250,6 +251,12 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
return backendProperties;
}
+ private void convertGlueToS3EndpointIfNeeded() {
+ if (this.endpoint.contains("glue")) {
+ this.endpoint = "s3." + this.region + ".amazonaws.com";
+ }
+ }
+
@Override
public AwsCredentialsProvider getAwsCredentialsProvider() {
AwsCredentialsProvider credentialsProvider =
super.getAwsCredentialsProvider();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
