This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 93989a62cf1 [opt](properties) remove old dlf properties (#55555)
93989a62cf1 is described below
commit 93989a62cf101903f97abc6064f73c021a88214d
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Sep 1 21:51:27 2025 -0700
[opt](properties) remove old dlf properties (#55555)
Remove old DLFProperties
---
.../java/org/apache/doris/catalog/HMSResource.java | 3 -
.../org/apache/doris/common/util/PrintableMap.java | 10 +-
.../lakesoul/LakeSoulExternalCatalog.java | 3 +-
.../property/ConnectorPropertiesUtils.java | 22 ++
.../datasource/property/ConnectorProperty.java | 2 +
.../datasource/property/PropertyConverter.java | 281 ---------------------
.../property/constants/DLFProperties.java | 60 -----
.../metastore/AliyunDLFBaseProperties.java | 7 +-
.../property/metastore/AliyunDLFProperties.java | 115 ---------
.../apache/doris/common/util/PrintableMapTest.java | 261 +++++++++++++++++++
.../datasource/property/PropertyConverterTest.java | 151 +----------
.../metastore/AliyunDLFBasePropertiesTest.java | 12 +
12 files changed, 310 insertions(+), 617 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
index 007cb4660e8..fccf2a6010b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HMSResource.java
@@ -19,7 +19,6 @@ package org.apache.doris.catalog;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.proc.BaseProcResult;
-import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.datasource.property.constants.HMSProperties;
import com.google.common.collect.ImmutableMap;
@@ -59,7 +58,6 @@ public class HMSResource extends Resource {
for (Map.Entry<String, String> kv : properties.entrySet()) {
replaceIfEffectiveValue(this.properties, kv.getKey(),
kv.getValue());
}
- this.properties =
PropertyConverter.convertToMetaProperties(this.properties);
super.modifyProperties(this.properties);
}
@@ -71,7 +69,6 @@ public class HMSResource extends Resource {
}
}
this.properties.putAll(properties);
- this.properties =
PropertyConverter.convertToMetaProperties(this.properties);
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
index 3ad8274d664..e035814f139 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PrintableMap.java
@@ -17,14 +17,15 @@
package org.apache.doris.common.util;
+import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
import org.apache.doris.datasource.property.constants.CosProperties;
-import org.apache.doris.datasource.property.constants.DLFProperties;
import org.apache.doris.datasource.property.constants.GCSProperties;
import org.apache.doris.datasource.property.constants.GlueProperties;
import org.apache.doris.datasource.property.constants.MCProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.datasource.property.constants.S3Properties;
+import org.apache.doris.datasource.property.metastore.AliyunDLFBaseProperties;
import com.google.common.collect.Sets;
@@ -64,12 +65,13 @@ public class PrintableMap<K, V> {
GCSProperties.SECRET_KEY,
CosProperties.SECRET_KEY,
GlueProperties.SECRET_KEY,
- MCProperties.SECRET_KEY,
- DLFProperties.SECRET_KEY));
+ MCProperties.SECRET_KEY));
+
+
SENSITIVE_KEY.addAll(ConnectorPropertiesUtils.getSensitiveKeys(AliyunDLFBaseProperties.class));
+
HIDDEN_KEY = Sets.newHashSet();
HIDDEN_KEY.addAll(S3Properties.Env.FS_KEYS);
HIDDEN_KEY.addAll(GlueProperties.META_KEYS);
- HIDDEN_KEY.addAll(DLFProperties.META_KEYS);
}
public PrintableMap(Map<K, V> map, String keyValueSeparator,
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java
index 15df1d3b567..7f8fcfb599a 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/lakesoul/LakeSoulExternalCatalog.java
@@ -21,7 +21,6 @@ import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
-import org.apache.doris.datasource.property.PropertyConverter;
import com.dmetasoul.lakesoul.meta.DBManager;
import com.dmetasoul.lakesoul.meta.DBUtil;
@@ -45,7 +44,7 @@ public class LakeSoulExternalCatalog extends ExternalCatalog {
public LakeSoulExternalCatalog(long catalogId, String name, String
resource, Map<String, String> props,
String comment) {
super(catalogId, name, InitCatalogLog.Type.LAKESOUL, comment);
- this.props = PropertyConverter.convertToMetaProperties(props);
+ this.props = props;
catalogProperty = new CatalogProperty(resource, props);
initLocalObjectsImpl();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorPropertiesUtils.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorPropertiesUtils.java
index 8b17bf3f217..35b8ce4e754 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorPropertiesUtils.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorPropertiesUtils.java
@@ -17,12 +17,14 @@
package org.apache.doris.datasource.property;
+import com.google.common.collect.Sets;
import org.apache.commons.lang3.StringUtils;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import java.util.Set;
/**
* Utility class for handling fields annotated with {@link ConnectorProperty}.
@@ -144,5 +146,25 @@ public class ConnectorPropertiesUtils {
throw new IllegalArgumentException("Unsupported property type: " +
targetType.getName());
}
+
+ /**
+ * Return the sensitive keys of the give properties
+ *
+ * @param clazz
+ * @return
+ */
+ public static Set<String> getSensitiveKeys(Class<?> clazz) {
+ Set<String> keys = Sets.newHashSet();
+ List<Field> supportedProps = getConnectorProperties(clazz);
+ for (Field field : supportedProps) {
+ ConnectorProperty anno =
field.getAnnotation(ConnectorProperty.class);
+ if (anno.sensitive()) {
+ for (String name : anno.names()) {
+ keys.add(name);
+ }
+ }
+ }
+ return keys;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorProperty.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorProperty.java
index 9fcaa5a6014..8cda6f99b8b 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorProperty.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/ConnectorProperty.java
@@ -27,4 +27,6 @@ public @interface ConnectorProperty {
boolean required() default true;
boolean supported() default true;
+
+ boolean sensitive() default false;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
index 1a9137ede5c..8529c5d8959 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/PropertyConverter.java
@@ -22,22 +22,16 @@ import
org.apache.doris.common.credentials.CloudCredentialWithEndpoint;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.datasource.CatalogMgr;
import org.apache.doris.datasource.InitCatalogLog.Type;
-import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
import org.apache.doris.datasource.property.constants.AzureProperties;
import org.apache.doris.datasource.property.constants.CosProperties;
-import org.apache.doris.datasource.property.constants.DLFProperties;
import org.apache.doris.datasource.property.constants.GCSProperties;
-import org.apache.doris.datasource.property.constants.GlueProperties;
-import org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
import org.apache.doris.datasource.property.constants.OssProperties;
import org.apache.doris.datasource.property.constants.PaimonProperties;
import org.apache.doris.datasource.property.constants.S3Properties;
-import com.aliyun.datalake.metastore.common.DataLakeConfig;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
-import com.amazonaws.glue.catalog.util.AWSGlueConfig;
import com.google.common.base.Strings;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.CosFileSystem;
@@ -50,7 +44,6 @@ import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.fs.s3a.S3AFileSystem;
import org.apache.hadoop.fs.s3a.TemporaryAWSCredentialsProvider;
import org.apache.hadoop.fs.s3a.auth.AssumedRoleCredentialProvider;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -62,55 +55,6 @@ public class PropertyConverter {
private static final Logger LOG =
LogManager.getLogger(PropertyConverter.class);
public static final String USE_PATH_STYLE = "use_path_style";
- /**
- * Convert properties defined at doris to metadata properties on Cloud
- * Step 1: convert and set cloud metadata properties and s3
properties
- * example:
- * glue.endpoint -> aws.glue.endpoint for Glue
- * -> s3.endpoint for S3
- * glue.access_key -> aws.glue.access-key for Glue
- * > s3.access_key for S3
- * Step 2: convert props to BE properties, put them all to
metaProperties
- * example:
- * s3.endpoint -> AWS_ENDPOINT
- * s3.access_key -> AWS_ACCESS_KEY
- * These properties will be used for catalog/resource, and persisted to
catalog/resource properties.
- * Some properties like AWS_XXX will be hidden, can find from HIDDEN_KEY
in PrintableMap
- * @see org.apache.doris.common.util.PrintableMap
- */
- public static Map<String, String> convertToMetaProperties(Map<String,
String> props) {
- Map<String, String> metaProperties = new HashMap<>();
- if (props.containsKey(GlueProperties.ENDPOINT)
- || props.containsKey(AWSGlueConfig.AWS_GLUE_ENDPOINT)) {
- CloudCredential credential = GlueProperties.getCredential(props);
- if (!credential.isWhole()) {
- credential = GlueProperties.getCompatibleCredential(props);
- }
- metaProperties = convertToGlueProperties(props, credential);
- } else if (props.containsKey(DLFProperties.ENDPOINT)
- || props.containsKey(DLFProperties.REGION)
- || props.containsKey(DataLakeConfig.CATALOG_ENDPOINT)) {
- metaProperties = convertToDLFProperties(props,
DLFProperties.getCredential(props));
- } else if (props.containsKey(S3Properties.Env.ENDPOINT)) {
- if (!hasS3Properties(props)) {
- // checkout env in the end
- // if meet AWS_XXX properties, convert to s3 properties
- return convertToS3EnvProperties(props,
S3Properties.getEnvironmentCredentialWithEndpoint(props), true);
- }
- }
- metaProperties.putAll(props);
- metaProperties.putAll(S3ClientBEProperties.getBeFSProperties(props));
- return metaProperties;
- }
-
- private static boolean hasS3Properties(Map<String, String> props) {
- return props.containsKey(ObsProperties.ENDPOINT)
- || props.containsKey(GCSProperties.ENDPOINT)
- || props.containsKey(OssProperties.ENDPOINT)
- || props.containsKey(CosProperties.ENDPOINT)
- || props.containsKey(MinioProperties.ENDPOINT);
- }
-
/**
* Convert properties defined at doris to FE S3 client properties
* Support other cloud client here.
@@ -409,229 +353,4 @@ public class PropertyConverter {
}
return convertToS3Properties(S3Properties.prefixToS3(props),
credential);
}
-
- private static Map<String, String> convertToDLFProperties(Map<String,
String> props, CloudCredential credential) {
- getPropertiesFromDLFConf(props);
- // if configure DLF properties in catalog properties, use them to
override config in hive-site.xml
- getPropertiesFromDLFProps(props, credential);
- props.put(DataLakeConfig.CATALOG_CREATE_DEFAULT_DB, "false");
- return props;
- }
-
- public static void getPropertiesFromDLFConf(Map<String, String> props) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Get properties from hive-site.xml");
- }
- // read properties from hive-site.xml.
- HiveConf hiveConf = new HiveConf();
- String metastoreType = hiveConf.get(HMSProperties.HIVE_METASTORE_TYPE);
- if (!HMSProperties.DLF_TYPE.equalsIgnoreCase(metastoreType)) {
- return;
- }
- String uid = props.get(DataLakeConfig.CATALOG_USER_ID);
- if (Strings.isNullOrEmpty(uid)) {
- throw new IllegalArgumentException("Required dlf property: " +
DataLakeConfig.CATALOG_USER_ID);
- }
- getOSSPropertiesFromDLFConf(props, hiveConf);
- }
-
- private static void getOSSPropertiesFromDLFConf(Map<String, String> props,
HiveConf hiveConf) {
- // get following properties from hive-site.xml
- // 1. region and endpoint. eg: cn-beijing
- String region = hiveConf.get(DataLakeConfig.CATALOG_REGION_ID);
- if (!Strings.isNullOrEmpty(region)) {
- // See: https://help.aliyun.com/document_detail/31837.html
- // And add "-internal" to access oss within vpc
- props.put(OssProperties.REGION, "oss-" + region);
- String publicAccess = hiveConf.get("dlf.catalog.accessPublic",
"false");
- props.put(OssProperties.ENDPOINT, getOssEndpoint(region,
Boolean.parseBoolean(publicAccess)));
- }
- // 2. ak and sk
- String ak = hiveConf.get(DataLakeConfig.CATALOG_ACCESS_KEY_ID);
- String sk = hiveConf.get(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET);
- if (!Strings.isNullOrEmpty(ak)) {
- props.put(OssProperties.ACCESS_KEY, ak);
- }
- if (!Strings.isNullOrEmpty(sk)) {
- props.put(OssProperties.SECRET_KEY, sk);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Get properties for oss in hive-site.xml: {}", props);
- }
- }
-
- private static void getPropertiesFromDLFProps(Map<String, String> props,
- CloudCredential credential) {
- String metastoreType = props.get(HMSProperties.HIVE_METASTORE_TYPE);
- if (!HMSProperties.DLF_TYPE.equalsIgnoreCase(metastoreType)) {
- return;
- }
- // convert to dlf client properties. not convert if origin key found.
- if (!props.containsKey(DataLakeConfig.CATALOG_USER_ID)) {
- props.put(DataLakeConfig.CATALOG_USER_ID,
props.get(DLFProperties.UID));
- String uid = props.get(DLFProperties.UID);
- if (Strings.isNullOrEmpty(uid)) {
- throw new IllegalArgumentException("Required dlf property: " +
DLFProperties.UID);
- }
-
- // region
- String region = props.get(DLFProperties.REGION);
- if (Strings.isNullOrEmpty(region)) {
- throw new IllegalArgumentException("Required dlf property: " +
DLFProperties.REGION);
- }
- props.put(DataLakeConfig.CATALOG_REGION_ID, region);
-
- // endpoint
- props.put(DataLakeConfig.CATALOG_ENDPOINT,
- props.getOrDefault(DLFProperties.ENDPOINT,
getDlfEndpointByRegion(region)));
-
- props.put(DataLakeConfig.CATALOG_PROXY_MODE,
props.getOrDefault(DLFProperties.PROXY_MODE, "DLF_ONLY"));
- props.put(DataLakeConfig.CATALOG_ACCESS_KEY_ID,
credential.getAccessKey());
- props.put(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET,
credential.getSecretKey());
- props.put(DLFProperties.Site.ACCESS_PUBLIC,
props.getOrDefault(DLFProperties.ACCESS_PUBLIC, "false"));
- }
- String uid = props.get(DataLakeConfig.CATALOG_USER_ID);
- if (Strings.isNullOrEmpty(uid)) {
- throw new IllegalArgumentException("Required dlf property: " +
DataLakeConfig.CATALOG_USER_ID);
- }
- if (!props.containsKey(DLFProperties.ENDPOINT)) {
- // just display DLFProperties in catalog, and hide DataLakeConfig
properties
- putNewPropertiesForCompatibility(props, credential);
- }
- // convert to oss property
- if (credential.isWhole()) {
- props.put(OssProperties.ACCESS_KEY, credential.getAccessKey());
- props.put(OssProperties.SECRET_KEY, credential.getSecretKey());
- }
- if (credential.isTemporary()) {
- props.put(OssProperties.SESSION_TOKEN,
credential.getSessionToken());
- }
- String publicAccess =
props.getOrDefault(DLFProperties.Site.ACCESS_PUBLIC, "false");
- String region = props.getOrDefault(DataLakeConfig.CATALOG_REGION_ID,
props.get(DLFProperties.REGION));
- if (!Strings.isNullOrEmpty(region)) {
- boolean hdfsEnabled =
Boolean.parseBoolean(props.getOrDefault(OssProperties.OSS_HDFS_ENABLED,
"false"));
- if (hdfsEnabled) {
- props.putIfAbsent("fs.oss.impl",
"com.aliyun.jindodata.oss.JindoOssFileSystem");
- props.put("fs.AbstractFileSystem.oss.impl",
"com.aliyun.jindodata.oss.OSS");
- props.putIfAbsent(OssProperties.REGION, region);
- // example: cn-shanghai.oss-dls.aliyuncs.com
- // from
https://www.alibabacloud.com/help/en/e-mapreduce/latest/oss-kusisurumen
- props.putIfAbsent(OssProperties.ENDPOINT, region +
".oss-dls.aliyuncs.com");
- } else {
- props.putIfAbsent(OssProperties.REGION, "oss-" + region);
- props.putIfAbsent(OssProperties.ENDPOINT,
getOssEndpoint(region, Boolean.parseBoolean(publicAccess)));
- }
- }
- }
-
- private static void putNewPropertiesForCompatibility(Map<String, String>
props, CloudCredential credential) {
- props.put(DLFProperties.UID,
props.get(DataLakeConfig.CATALOG_USER_ID));
- String endpoint = props.get(DataLakeConfig.CATALOG_ENDPOINT);
- props.put(DLFProperties.ENDPOINT, endpoint);
- props.put(DLFProperties.REGION,
props.getOrDefault(DataLakeConfig.CATALOG_REGION_ID,
- S3Properties.getRegionOfEndpoint(endpoint)));
- props.put(DLFProperties.PROXY_MODE,
props.getOrDefault(DataLakeConfig.CATALOG_PROXY_MODE, "DLF_ONLY"));
- props.put(DLFProperties.ACCESS_KEY, credential.getAccessKey());
- props.put(DLFProperties.SECRET_KEY, credential.getSecretKey());
- props.put(DLFProperties.ACCESS_PUBLIC,
props.getOrDefault(DLFProperties.Site.ACCESS_PUBLIC, "false"));
- }
-
- private static String getOssEndpoint(String region, boolean publicAccess) {
- String prefix = "oss-";
- String suffix = ".aliyuncs.com";
- if (!publicAccess) {
- suffix = "-internal" + suffix;
- }
- return prefix + region + suffix;
- }
-
- private static String getDlfEndpointByRegion(String region) {
- return "dlf-vpc." + region + ".aliyuncs.com";
- }
-
- private static Map<String, String> convertToGlueProperties(Map<String,
String> props, CloudCredential credential) {
- // convert doris glue property to glue properties, s3 client property
and BE property
- String metastoreType = props.get(HMSProperties.HIVE_METASTORE_TYPE);
- String icebergType =
props.get(IcebergExternalCatalog.ICEBERG_CATALOG_TYPE);
- boolean isGlueIceberg =
IcebergExternalCatalog.ICEBERG_GLUE.equals(icebergType);
- if (!HMSProperties.GLUE_TYPE.equalsIgnoreCase(metastoreType) &&
!isGlueIceberg) {
- return props;
- }
- if (isGlueIceberg) {
- // glue ak sk for iceberg
- props.putIfAbsent(GlueProperties.ACCESS_KEY,
credential.getAccessKey());
- props.putIfAbsent(GlueProperties.SECRET_KEY,
credential.getSecretKey());
- props.putIfAbsent(GlueProperties.CLIENT_CREDENTIALS_PROVIDER,
-
"com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x");
- props.putIfAbsent(GlueProperties.CLIENT_CREDENTIALS_PROVIDER_AK,
credential.getAccessKey());
- props.putIfAbsent(GlueProperties.CLIENT_CREDENTIALS_PROVIDER_SK,
credential.getSecretKey());
- }
- // set glue client metadata
- if (props.containsKey(GlueProperties.ENDPOINT)) {
- String endpoint = props.get(GlueProperties.ENDPOINT);
- props.put(AWSGlueConfig.AWS_GLUE_ENDPOINT, endpoint);
- String region = S3Properties.getRegionOfEndpoint(endpoint);
- props.put(AWSGlueConfig.AWS_REGION, region);
- if (credential.isWhole()) {
- props.put(AWSGlueConfig.AWS_GLUE_ACCESS_KEY,
credential.getAccessKey());
- props.put(AWSGlueConfig.AWS_GLUE_SECRET_KEY,
credential.getSecretKey());
- }
- if (credential.isTemporary()) {
- props.put(AWSGlueConfig.AWS_GLUE_SESSION_TOKEN,
credential.getSessionToken());
- }
- } else {
- // compatible with old version, deprecated in the future version
- // put GlueProperties to map if origin key found.
- if (props.containsKey(AWSGlueConfig.AWS_GLUE_ENDPOINT)) {
- String endpoint = props.get(AWSGlueConfig.AWS_GLUE_ENDPOINT);
- props.put(GlueProperties.ENDPOINT, endpoint);
- if (props.containsKey(AWSGlueConfig.AWS_GLUE_ACCESS_KEY)) {
- props.put(GlueProperties.ACCESS_KEY,
props.get(AWSGlueConfig.AWS_GLUE_ACCESS_KEY));
- }
- if (props.containsKey(AWSGlueConfig.AWS_GLUE_SECRET_KEY)) {
- props.put(GlueProperties.SECRET_KEY,
props.get(AWSGlueConfig.AWS_GLUE_SECRET_KEY));
- }
- if (props.containsKey(AWSGlueConfig.AWS_GLUE_SESSION_TOKEN)) {
- props.put(GlueProperties.SESSION_TOKEN,
props.get(AWSGlueConfig.AWS_GLUE_SESSION_TOKEN));
- }
- }
- }
- // set s3 client credential
- // https://docs.aws.amazon.com/general/latest/gr/s3.html
- // Convert:
- // (
- // "glue.region" = "us-east-1",
- // "glue.access_key" = "xx",
- // "glue.secret_key" = "yy"
- // )
- // To:
- // (
- // "s3.region" = "us-east-1",
- // "s3.endpoint" = "s3.us-east-1.amazonaws.com"
- // "s3.access_key" = "xx",
- // "s3.secret_key" = "yy"
- // )
- String endpoint = props.get(GlueProperties.ENDPOINT);
- String region = S3Properties.getRegionOfEndpoint(endpoint);
- if (!Strings.isNullOrEmpty(region)) {
- props.put(S3Properties.REGION, region);
- String suffix = ".amazonaws.com";
- if (endpoint.endsWith(".amazonaws.com.cn")) {
- suffix = ".amazonaws.com.cn";
- }
- String s3Endpoint = "s3." + region + suffix;
- if (isGlueIceberg) {
- s3Endpoint = "https://"; + s3Endpoint;
- }
- props.put(S3Properties.ENDPOINT, s3Endpoint);
- }
- if (credential.isWhole()) {
- props.put(S3Properties.ACCESS_KEY, credential.getAccessKey());
- props.put(S3Properties.SECRET_KEY, credential.getSecretKey());
- }
- if (credential.isTemporary()) {
- props.put(S3Properties.SESSION_TOKEN,
credential.getSessionToken());
- }
- return props;
- }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
deleted file mode 100644
index d8a3b7d477a..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/constants/DLFProperties.java
+++ /dev/null
@@ -1,60 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.constants;
-
-import org.apache.doris.common.credentials.CloudCredential;
-
-import com.aliyun.datalake.metastore.common.DataLakeConfig;
-import com.google.common.collect.ImmutableList;
-
-import java.util.List;
-import java.util.Map;
-
-public class DLFProperties extends BaseProperties {
- public static final String ACCESS_PUBLIC = "dlf.access.public";
- public static final String UID = "dlf.uid";
- public static final String PROXY_MODE = "dlf.proxy.mode";
- public static final String ENDPOINT = "dlf.endpoint";
- public static final String ACCESS_KEY = "dlf.access_key";
- public static final String SECRET_KEY = "dlf.secret_key";
- public static final String REGION = "dlf.region";
- public static final String SESSION_TOKEN = "dlf.session_token";
- public static final List<String> META_KEYS =
ImmutableList.of(DataLakeConfig.CATALOG_PROXY_MODE,
- Site.ACCESS_PUBLIC,
- DataLakeConfig.CATALOG_USER_ID,
- DataLakeConfig.CATALOG_CREATE_DEFAULT_DB,
- DataLakeConfig.CATALOG_ENDPOINT,
- DataLakeConfig.CATALOG_REGION_ID,
- DataLakeConfig.CATALOG_SECURITY_TOKEN,
- DataLakeConfig.CATALOG_ACCESS_KEY_SECRET,
- DataLakeConfig.CATALOG_ACCESS_KEY_ID);
-
- public static class Site {
- public static final String ACCESS_PUBLIC = "dlf.catalog.accessPublic";
- }
-
- public static CloudCredential getCredential(Map<String, String> props) {
- CloudCredential credential = getCloudCredential(props, ACCESS_KEY,
SECRET_KEY, SESSION_TOKEN);
- if (!credential.isWhole()) {
- credential = getCloudCredential(props,
DataLakeConfig.CATALOG_ACCESS_KEY_ID,
- DataLakeConfig.CATALOG_ACCESS_KEY_SECRET,
- DataLakeConfig.CATALOG_SECURITY_TOKEN);
- }
- return credential;
- }
-}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
index fc68e74efa4..8b3d6735fa7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
@@ -34,12 +34,14 @@ public class AliyunDLFBaseProperties {
protected String dlfAccessKey = "";
@ConnectorProperty(names = {"dlf.secret_key",
"dlf.catalog.accessKeySecret"},
- description = "The secret key of the Aliyun DLF.")
+ description = "The secret key of the Aliyun DLF.",
+ sensitive = true)
protected String dlfSecretKey = "";
@ConnectorProperty(names = {"dlf.session_token",
"dlf.catalog.sessionToken"},
required = false,
- description = "The session token of the Aliyun DLF.")
+ description = "The session token of the Aliyun DLF.",
+ sensitive = true)
protected String dlfSessionToken = "";
@ConnectorProperty(names = {"dlf.region"},
@@ -79,7 +81,6 @@ public class AliyunDLFBaseProperties {
}
private ParamRules buildRules() {
-
return new ParamRules()
.require(dlfAccessKey, "dlf.access_key is required")
.require(dlfSecretKey, "dlf.secret_key is required");
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java
deleted file mode 100644
index eabb383a5f5..00000000000
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFProperties.java
+++ /dev/null
@@ -1,115 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.datasource.property.metastore;
-
-import org.apache.doris.datasource.property.ConnectorProperty;
-
-import com.google.common.base.Strings;
-import com.google.common.collect.Maps;
-import lombok.Getter;
-import org.apache.paimon.options.Options;
-
-import java.util.Map;
-
-public class AliyunDLFProperties extends MetastoreProperties {
-
- @ConnectorProperty(names = {"dlf.access_key", "dlf.catalog.accessKeyId"},
- description = "The access key of the Aliyun DLF.")
- public String dlfAccessKey = "";
-
- @ConnectorProperty(names = {"dlf.secret_key",
"dlf.catalog.accessKeySecret"},
- description = "The secret key of the Aliyun DLF.")
- private String dlfSecretKey = "";
-
- @ConnectorProperty(names = {"dlf.region"},
- description = "The region of the Aliyun DLF.")
- private String dlfRegion = "";
-
- @ConnectorProperty(names = {"dlf.endpoint", "dlf.catalog.endpoint"},
- required = false,
- description = "The region of the Aliyun DLF.")
- private String dlfEndpoint = "";
-
- @ConnectorProperty(names = {"dlf.uid", "dlf.catalog.uid"},
- description = "The uid of the Aliyun DLF.")
- private String dlfUid = "";
-
- @ConnectorProperty(names = {"dlf.access.public",
"dlf.catalog.accessPublic"},
- required = false,
- description = "Enable public access to Aliyun DLF.")
- private String dlfAccessPublic = "false";
-
- private static final String DLF_PREFIX = "dlf.";
-
- @Getter
- private final Map<String, String> otherDlfProps = Maps.newHashMap();
-
- private Map<String, String> dlfConnectProps = Maps.newHashMap();
-
- public AliyunDLFProperties(Map<String, String> origProps) {
- super(Type.DLF, origProps);
- }
-
- @Override
- public void initNormalizeAndCheckProps() {
- super.initNormalizeAndCheckProps();
- // Other properties that start with "dlf." will be saved in
otherDlfProps,
- // and passed to the DLF client.
- for (Map.Entry<String, String> entry : origProps.entrySet()) {
- if (entry.getKey().startsWith(DLF_PREFIX) &&
!matchedProperties.containsKey(entry.getKey())) {
- otherDlfProps.put(entry.getKey(), entry.getValue());
- }
- }
- initDlfConnectProps();
- }
-
- private void initDlfConnectProps() {
- dlfConnectProps.put("dlf.catalog.region", dlfRegion);
- dlfConnectProps.put("dlf.catalog.endpoint",
getEndpointOrFromRegion(dlfEndpoint, dlfRegion, dlfAccessPublic));
- dlfConnectProps.put("dlf.catalog.proxyMode", "DLF_ONLY");
- dlfConnectProps.put("dlf.catalog.accessKeyId", dlfAccessKey);
- dlfConnectProps.put("dlf.catalog.accessKeySecret", dlfSecretKey);
- dlfConnectProps.put("dlf.catalog.accessPublic", dlfAccessPublic);
- dlfConnectProps.put("dlf.catalog.uid", dlfUid);
- dlfConnectProps.put("dlf.catalog.createDefaultDBIfNotExist", "false");
- otherDlfProps.forEach((dlfConnectProps::put));
- }
-
- public void toPaimonOptions(Options options) {
- // See DataLakeConfig.java for property keys
- dlfConnectProps.forEach(options::set);
- }
-
- private String getEndpointOrFromRegion(String endpoint, String region,
String dlfAccessPublic) {
- if (!Strings.isNullOrEmpty(endpoint)) {
- return endpoint;
- } else {
- //
https://www.alibabacloud.com/help/en/dlf/dlf-1-0/regions-and-endpoints
- if ("true".equalsIgnoreCase(dlfAccessPublic)) {
- return "dlf." + region + ".aliyuncs.com";
- } else {
- return "dlf-vpc." + region + ".aliyuncs.com";
- }
- }
- }
-
- @Override
- protected String getResourceConfigPropName() {
- return "dlf.resource_config";
- }
-}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/common/util/PrintableMapTest.java
b/fe/fe-core/src/test/java/org/apache/doris/common/util/PrintableMapTest.java
new file mode 100644
index 00000000000..0514cfedb9a
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/common/util/PrintableMapTest.java
@@ -0,0 +1,261 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.util;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+public class PrintableMapTest {
+
+ @Test
+ public void testSensitiveKeysContainAliyunDLFProperties() {
+ // Verify that SENSITIVE_KEY contains sensitive keys from
AliyunDLFBaseProperties
+ // These keys are added via
ConnectorPropertiesUtils.getSensitiveKeys(AliyunDLFBaseProperties.class)
+
+ // Verify sensitive keys from AliyunDLFBaseProperties
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("dlf.secret_key"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("dlf.catalog.accessKeySecret"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("dlf.session_token"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("dlf.catalog.sessionToken"));
+
+ // Verify other common sensitive keys
+ Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("password"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("kerberos_keytab_content"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("bos_secret_accesskey"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("jdbc.password"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("elasticsearch.password"));
+
+ // Verify cloud storage related sensitive keys (these are constants
added in static initialization block)
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("s3.secret_key"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("AWS_SECRET_KEY"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("obs.secret_key"));
+
Assertions.assertTrue(PrintableMap.SENSITIVE_KEY.contains("oss.secret_key"));
+ }
+
+ @Test
+ public void testBasicConstructor() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("key1", "value1");
+ testMap.put("key2", "value2");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", true, false);
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("\"key1\" = \"value1\""));
+ Assertions.assertTrue(result.contains("\"key2\" = \"value2\""));
+ }
+
+ @Test
+ public void testConstructorWithEntryDelimiter() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("key1", "value1");
+ testMap.put("key2", "value2");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, ":", false, false, ";");
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("key1 : value1"));
+ Assertions.assertTrue(result.contains("key2 : value2"));
+ Assertions.assertTrue(result.contains(";"));
+ }
+
+ @Test
+ public void testConstructorWithHidePassword() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("username", "admin");
+ testMap.put("password", "secret123");
+ testMap.put("dlf.secret_key", "dlf_secret");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false, true);
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("username = admin"));
+ Assertions.assertTrue(result.contains("password = " +
PrintableMap.PASSWORD_MASK));
+ Assertions.assertTrue(result.contains("dlf.secret_key = " +
PrintableMap.PASSWORD_MASK));
+ }
+
+ @Test
+ public void testConstructorWithSorted() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("zebra", "value3");
+ testMap.put("apple", "value1");
+ testMap.put("banana", "value2");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false, true, true);
+ String result = printableMap.toString();
+
+ // Verify sorting (descending order)
+ int zebraIndex = result.indexOf("zebra");
+ int bananaIndex = result.indexOf("banana");
+ int appleIndex = result.indexOf("apple");
+
+ Assertions.assertTrue(zebraIndex < bananaIndex);
+ Assertions.assertTrue(bananaIndex < appleIndex);
+ }
+
+ @Test
+ public void testWithQuotation() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("key1", "value1");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", true, false);
+ String result = printableMap.toString();
+
+ Assertions.assertEquals("\"key1\" = \"value1\"", result);
+ }
+
+ @Test
+ public void testWithoutQuotation() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("key1", "value1");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false);
+ String result = printableMap.toString();
+
+ Assertions.assertEquals("key1 = value1", result);
+ }
+
+ @Test
+ public void testWithWrap() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("key1", "value1");
+ testMap.put("key2", "value2");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, true);
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("\n"));
+ Assertions.assertTrue(result.contains("key1 = value1"));
+ Assertions.assertTrue(result.contains("key2 = value2"));
+ }
+
+ @Test
+ public void testHidePasswordWithSensitiveKeys() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("normal_key", "normal_value");
+ testMap.put("password", "secret_password");
+ testMap.put("dlf.secret_key", "dlf_secret_value");
+ testMap.put("s3.secret_key", "s3_secret_value");
+ testMap.put("kerberos_keytab_content", "kerberos_content");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false, true);
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("normal_key = normal_value"));
+ Assertions.assertTrue(result.contains("password = " +
PrintableMap.PASSWORD_MASK));
+ Assertions.assertTrue(result.contains("dlf.secret_key = " +
PrintableMap.PASSWORD_MASK));
+ Assertions.assertTrue(result.contains("s3.secret_key = " +
PrintableMap.PASSWORD_MASK));
+ Assertions.assertTrue(result.contains("kerberos_keytab_content = " +
PrintableMap.PASSWORD_MASK));
+ }
+
+ @Test
+ public void testCaseInsensitiveSensitiveKeys() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("PASSWORD", "secret_password");
+ testMap.put("Password", "another_secret");
+ testMap.put("password", "third_secret");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false, true);
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("PASSWORD = " +
PrintableMap.PASSWORD_MASK));
+ Assertions.assertTrue(result.contains("Password = " +
PrintableMap.PASSWORD_MASK));
+ Assertions.assertTrue(result.contains("password = " +
PrintableMap.PASSWORD_MASK));
+ }
+
+ @Test
+ public void testAdditionalHiddenKeys() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("visible_key", "visible_value");
+ testMap.put("hidden_key", "hidden_value");
+ testMap.put("another_hidden", "another_value");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false);
+ Set<String> hiddenKeys = new java.util.HashSet<>();
+ hiddenKeys.add("hidden_key");
+ hiddenKeys.add("another_hidden");
+ printableMap.setAdditionalHiddenKeys(hiddenKeys);
+
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("visible_key = visible_value"));
+ Assertions.assertFalse(result.contains("hidden_key"));
+ Assertions.assertFalse(result.contains("another_hidden"));
+ }
+
+ @Test
+ public void testNullMap() {
+ PrintableMap<String, String> printableMap = new PrintableMap<>(null,
"=", false, false);
+ String result = printableMap.toString();
+
+ Assertions.assertEquals("", result);
+ }
+
+ @Test
+ public void testEmptyMap() {
+ Map<String, String> testMap = new HashMap<>();
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false);
+ String result = printableMap.toString();
+
+ Assertions.assertEquals("", result);
+ }
+
+ @Test
+ public void testSingleEntry() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("single_key", "single_value");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "=", false, false);
+ String result = printableMap.toString();
+
+ Assertions.assertEquals("single_key = single_value", result);
+ }
+
+ @Test
+ public void testMultipleEntriesWithCustomDelimiter() {
+ Map<String, String> testMap = new HashMap<>();
+ testMap.put("key1", "value1");
+ testMap.put("key2", "value2");
+ testMap.put("key3", "value3");
+
+ PrintableMap<String, String> printableMap = new
PrintableMap<>(testMap, "->", false, false, "|");
+ String result = printableMap.toString();
+
+ Assertions.assertTrue(result.contains("key1 -> value1"));
+ Assertions.assertTrue(result.contains("key2 -> value2"));
+ Assertions.assertTrue(result.contains("key3 -> value3"));
+ Assertions.assertTrue(result.contains("|"));
+ Assertions.assertFalse(result.contains(","));
+ }
+
+ @Test
+ public void testHiddenKeysFromS3AndGlue() {
+ // Verify that HIDDEN_KEY contains S3 and Glue related keys
+ // These keys are added in the static initialization block
+ Assertions.assertFalse(PrintableMap.HIDDEN_KEY.isEmpty());
+ }
+
+ @Test
+ public void testPasswordMaskConstant() {
+ Assertions.assertEquals("*XXX", PrintableMap.PASSWORD_MASK);
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
index 6318a00dea1..374df3f311c 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/PropertyConverterTest.java
@@ -23,16 +23,11 @@ import org.apache.doris.catalog.Resource;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.FeMetaVersion;
-import org.apache.doris.common.util.PrintableMap;
-import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalCatalog;
import org.apache.doris.datasource.iceberg.IcebergExternalCatalog;
-import org.apache.doris.datasource.iceberg.IcebergGlueExternalCatalog;
import org.apache.doris.datasource.maxcompute.MaxComputeExternalCatalog;
import org.apache.doris.datasource.property.constants.CosProperties;
-import org.apache.doris.datasource.property.constants.DLFProperties;
import org.apache.doris.datasource.property.constants.GCSProperties;
-import org.apache.doris.datasource.property.constants.GlueProperties;
import org.apache.doris.datasource.property.constants.HMSProperties;
import org.apache.doris.datasource.property.constants.MinioProperties;
import org.apache.doris.datasource.property.constants.ObsProperties;
@@ -430,7 +425,8 @@ public class PropertyConverterTest extends
TestWithFeService {
((CreateCatalogCommand) logicalPlan).run(connectContext, null);
}
- HMSExternalCatalog catalogNew = (HMSExternalCatalog)
Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
+ HMSExternalCatalog catalogNew = (HMSExternalCatalog)
Env.getCurrentEnv()
+ .getCatalogMgr().getCatalog(catalogName);
Map<String, String> propertiesNew = catalogNew.getProperties();
Assertions.assertEquals(22, propertiesNew.size());
Assertions.assertEquals("s3.us-east-1.amazonaws.com.cn",
propertiesNew.get(S3Properties.ENDPOINT));
@@ -654,147 +650,4 @@ public class PropertyConverterTest extends
TestWithFeService {
Assertions.assertEquals("region",
beProperties.get(S3Properties.Env.REGION));
Assertions.assertEquals("false",
beProperties.get(PropertyConverter.USE_PATH_STYLE));
}
-
- @Test
- public void testMetaPropertiesConvertor() {
- // test region parser
-
Assertions.assertNull(S3Properties.getRegionOfEndpoint("http://192.168.2.30:9099/com.region.test/dir";));
- Assertions.assertEquals("cn-beijing",
-
S3Properties.getRegionOfEndpoint("http://dlf.cn-beijing.aliyuncs.com/com.region.test/dir";));
- Assertions.assertEquals("oss-cn-beijing",
-
S3Properties.getRegionOfEndpoint("http://oss-cn-beijing.aliyuncs.com/com.region.test/dir";));
- Assertions.assertEquals("us-east-1",
-
S3Properties.getRegionOfEndpoint("http://s3.us-east-1.amazonaws.com/com.region.test/dir";));
-
- //1. dlf
- Map<String, String> props = new HashMap<>();
- // iceberg.catalog.type
- props.put("type", "hms");
- props.put("hive.metastore.type", "dlf");
- props.put(DLFProperties.PROXY_MODE, "DLF_ONLY");
- props.put(DLFProperties.ENDPOINT, "dlf.cn-beijing.aliyuncs.com");
- props.put(DLFProperties.UID, "20239444");
- props.put(DLFProperties.ACCESS_KEY, "akk");
- props.put(DLFProperties.SECRET_KEY, "skk");
- props.put(DLFProperties.REGION, "cn-beijing");
- props.put(DLFProperties.ACCESS_PUBLIC, "false");
- Map<String, String> res =
PropertyConverter.convertToMetaProperties(new HashMap<>(props));
- Assertions.assertEquals(26, res.size());
- Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
- Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
- Assertions.assertEquals("akk",
res.get(DataLakeConfig.CATALOG_ACCESS_KEY_ID));
- Assertions.assertEquals("skk",
res.get(DataLakeConfig.CATALOG_ACCESS_KEY_SECRET));
- Assertions.assertEquals("dlf.cn-beijing.aliyuncs.com",
res.get(DataLakeConfig.CATALOG_ENDPOINT));
- Assertions.assertEquals("oss-cn-beijing-internal.aliyuncs.com",
res.get(S3Properties.Env.ENDPOINT));
- Assertions.assertEquals("cn-beijing",
res.get(DataLakeConfig.CATALOG_REGION_ID));
- Assertions.assertEquals("oss-cn-beijing",
res.get(S3Properties.Env.REGION));
-
- props.put(DLFProperties.ACCESS_PUBLIC, "true");
- res = PropertyConverter.convertToMetaProperties(new HashMap<>(props));
- Assertions.assertEquals(26, res.size());
- Assertions.assertEquals("oss-cn-beijing.aliyuncs.com",
res.get(S3Properties.Env.ENDPOINT));
- Assertions.assertEquals("oss-cn-beijing",
res.get(S3Properties.Env.REGION));
-
- props.put(OssProperties.OSS_HDFS_ENABLED, "true");
- res = PropertyConverter.convertToMetaProperties(new HashMap<>(props));
- Assertions.assertEquals(29, res.size());
- Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem",
res.get("fs.oss.impl"));
- Assertions.assertEquals("com.aliyun.jindodata.oss.OSS",
res.get("fs.AbstractFileSystem.oss.impl"));
- Assertions.assertEquals("false",
res.get(DataLakeConfig.CATALOG_CREATE_DEFAULT_DB));
- Assertions.assertEquals("cn-beijing",
res.get(S3Properties.Env.REGION));
-
- // 2. glue
- Map<String, String> props2 = new HashMap<>();
- props2.put("hive.metastore.type", "glue");
- props2.put("aws.glue.endpoint", "glue.us-east-1.amazonaws.com");
- props2.put("aws.glue.access-key", "akk");
- props2.put("aws.glue.secret-key", "skk");
- props2.put("aws.region", "us-east-1");
- res = PropertyConverter.convertToMetaProperties(props2);
- Assertions.assertEquals(17, res.size());
- Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
- Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
- Assertions.assertEquals("s3.us-east-1.amazonaws.com",
res.get(S3Properties.Env.ENDPOINT));
- Assertions.assertEquals("us-east-1", res.get(S3Properties.Env.REGION));
-
- Map<String, String> props3 = new HashMap<>();
- props3.put("hive.metastore.type", "glue");
- props3.put(GlueProperties.ENDPOINT, "glue.us-east-1.amazonaws.com");
- props3.put(GlueProperties.ACCESS_KEY, "akk");
- props3.put(GlueProperties.SECRET_KEY, "skk");
- res = PropertyConverter.convertToMetaProperties(props3);
- Assertions.assertEquals(17, res.size());
- Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
- Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
- Assertions.assertEquals("s3.us-east-1.amazonaws.com",
res.get(S3Properties.Env.ENDPOINT));
- Assertions.assertEquals("us-east-1", res.get(S3Properties.Env.REGION));
-
- // 3. s3 env
- Map<String, String> props4 = new HashMap<>();
- props4.put("hive.metastore.type", "hms");
- props4.put(S3Properties.Env.ENDPOINT, "s3.us-west-2.amazonaws.com");
- props4.put(S3Properties.Env.ACCESS_KEY, "akk");
- props4.put(S3Properties.Env.SECRET_KEY, "skk");
- res = PropertyConverter.convertToMetaProperties(new HashMap<>(props4));
- Assertions.assertEquals(9, res.size());
- Assertions.assertEquals("akk", res.get(S3Properties.Env.ACCESS_KEY));
- Assertions.assertEquals("skk", res.get(S3Properties.Env.SECRET_KEY));
- Assertions.assertEquals("s3.us-west-2.amazonaws.com",
res.get(S3Properties.Env.ENDPOINT));
- Assertions.assertEquals("us-west-2", res.get(S3Properties.Env.REGION));
-
- props4.put(S3Properties.Env.ENDPOINT, "http://172.23.56.19:9033";);
- res = PropertyConverter.convertToMetaProperties(new HashMap<>(props4));
- Assertions.assertEquals(9, res.size());
- Assertions.assertEquals("http://172.23.56.19:9033";,
res.get(S3Properties.Env.ENDPOINT));
- Assertions.assertEquals("us-east-1", res.get(S3Properties.Env.REGION));
-
- props4.put(S3Properties.Env.REGION, "north");
- res = PropertyConverter.convertToMetaProperties(new HashMap<>(props4));
- Assertions.assertEquals(9, res.size());
- Assertions.assertEquals("north", res.get(S3Properties.Env.REGION));
- }
-
- @Test
- public void testGluePropertiesConvertor() throws Exception {
- Map<String, String> originProps = Maps.newHashMap();
- originProps.put(GlueProperties.ACCESS_KEY, "ak");
- originProps.put(GlueProperties.SECRET_KEY, "sk");
- originProps.put(GlueProperties.ENDPOINT,
"https://glue.us-east-1.amazonaws.com";);
- originProps.put("type", "iceberg");
- originProps.put("iceberg.catalog.type", "glue");
-
- Map<String, String> convertedProps =
PropertyConverter.convertToMetaProperties(originProps);
- System.out.println(convertedProps);
-
Assertions.assertEquals("com.amazonaws.glue.catalog.credentials.ConfigurationAWSCredentialsProvider2x",
-
convertedProps.get(GlueProperties.CLIENT_CREDENTIALS_PROVIDER));
- Assertions.assertEquals("ak",
convertedProps.get(GlueProperties.CLIENT_CREDENTIALS_PROVIDER_AK));
- Assertions.assertEquals("sk",
convertedProps.get(GlueProperties.CLIENT_CREDENTIALS_PROVIDER_SK));
-
- String createIceGlue = "CREATE CATALOG iceglue PROPERTIES (\n"
- + " \"type\"=\"iceberg\",\n"
- + " \"iceberg.catalog.type\" = \"glue\",\n"
- + " \"glue.endpoint\" =
\"https://glue.us-east-1.amazonaws.com/\",\n";
- + " \"glue.access_key\" = \"ak123\",\n"
- + " \"glue.secret_key\" = \"sk123\"\n"
- + ");";
-
- NereidsParser nereidsParser = new NereidsParser();
- LogicalPlan logicalPlan = nereidsParser.parseSingle(createIceGlue);
- if (logicalPlan instanceof CreateCatalogCommand) {
- ((CreateCatalogCommand) logicalPlan).run(connectContext, null);
- }
-
- IcebergExternalCatalog icebergExternalCatalog =
(IcebergExternalCatalog)
Env.getCurrentEnv().getCatalogMgr().getCatalog("iceglue");
- Assertions.assertTrue(icebergExternalCatalog instanceof
IcebergGlueExternalCatalog);
- IcebergGlueExternalCatalog glueCatalog = (IcebergGlueExternalCatalog)
icebergExternalCatalog;
-
- PrintableMap<String, String> printableMap = new
PrintableMap<>(glueCatalog.getProperties(), "=", true, true,
- true, true);
-
printableMap.setAdditionalHiddenKeys(ExternalCatalog.HIDDEN_PROPERTIES);
- String result = printableMap.toString();
- System.out.println(result);
-
Assertions.assertTrue(!result.contains(GlueProperties.CLIENT_CREDENTIALS_PROVIDER));
-
Assertions.assertTrue(!result.contains(GlueProperties.CLIENT_CREDENTIALS_PROVIDER_AK));
-
Assertions.assertTrue(!result.contains(GlueProperties.CLIENT_CREDENTIALS_PROVIDER_SK));
- }
}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AliyunDLFBasePropertiesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AliyunDLFBasePropertiesTest.java
index 8e6324b0e3f..ba28c4caab3 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AliyunDLFBasePropertiesTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AliyunDLFBasePropertiesTest.java
@@ -17,6 +17,7 @@
package org.apache.doris.datasource.property.metastore;
+import org.apache.doris.datasource.property.ConnectorPropertiesUtils;
import
org.apache.doris.datasource.property.storage.exception.StoragePropertiesException;
import org.junit.jupiter.api.Assertions;
@@ -24,6 +25,7 @@ import org.junit.jupiter.api.Test;
import java.util.HashMap;
import java.util.Map;
+import java.util.Set;
public class AliyunDLFBasePropertiesTest {
@@ -102,4 +104,14 @@ public class AliyunDLFBasePropertiesTest {
);
Assertions.assertTrue(ex.getMessage().contains("dlf.secret_key is
required"));
}
+
+ @Test
+ void testGetSensitiveKeys() {
+ Set<String> keys =
ConnectorPropertiesUtils.getSensitiveKeys(AliyunDLFBaseProperties.class);
+ System.out.println(keys);
+ Assertions.assertTrue(keys.contains("dlf.catalog.sessionToken"));
+ Assertions.assertTrue(keys.contains("dlf.catalog.accessKeySecret"));
+ Assertions.assertTrue(keys.contains("dlf.secret_key"));
+ Assertions.assertTrue(keys.contains("dlf.session_token"));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
