This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new ef2c200b6f2 branch-3.1: [chore](param-refactor)Optimize catalog
parameter handling and add Hive hadoop.username mapping #54551 (#54753)
ef2c200b6f2 is described below
commit ef2c200b6f2cfb8a1dae9fc94976e5a6cdccc88f
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Thu Aug 14 17:39:26 2025 +0800
branch-3.1: [chore](param-refactor)Optimize catalog parameter handling and
add Hive hadoop.username mapping #54551 (#54753)
Cherry-picked from #54551
Co-authored-by: Calvin Kirs <[email protected]>
---
.../security/authentication/AuthenticationConfig.java | 3 +--
.../metastore/AWSGlueMetaStoreBaseProperties.java | 3 ++-
.../property/metastore/AWSGlueMetaStoreProperties.java | 1 -
.../property/metastore/AliyunDLFBaseProperties.java | 3 ++-
.../property/metastore/HMSBaseProperties.java | 18 ++++++++++++++++--
.../property/metastore/HMSGlueMetaStoreProperties.java | 1 -
.../datasource/property/metastore/HMSProperties.java | 3 ---
.../metastore/IcebergGlueMetaStoreProperties.java | 4 ----
.../metastore/IcebergHMSMetaStoreProperties.java | 1 -
.../metastore/IcebergS3TablesMetaStoreProperties.java | 1 -
.../metastore/PaimonFileSystemMetaStoreProperties.java | 2 +-
.../metastore/PaimonHMSMetaStoreProperties.java | 1 -
.../datasource/property/storage/S3Properties.java | 1 +
.../kerberos/test_single_hive_kerberos.groovy | 4 ++++
.../kerberos/test_two_hive_kerberos.groovy | 4 ++++
15 files changed, 31 insertions(+), 19 deletions(-)
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
index 1277aec98f1..41804037a38 100644
---
a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
+++
b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
@@ -30,8 +30,7 @@ public abstract class AuthenticationConfig {
public static String HADOOP_USER_NAME = "hadoop.username";
public static String HADOOP_KERBEROS_PRINCIPAL =
"hadoop.kerberos.principal";
public static String HADOOP_KERBEROS_KEYTAB = "hadoop.kerberos.keytab";
- public static String HIVE_KERBEROS_PRINCIPAL =
"hive.metastore.kerberos.principal";
- public static String HIVE_KERBEROS_KEYTAB =
"hive.metastore.kerberos.keytab.file";
+ public static String HADOOP_SECURITY_AUTH_TO_LOCAL =
"hadoop.security.auth_to_local";
public static String DORIS_KRB5_DEBUG = "doris.krb5.debug";
private static final String DEFAULT_HADOOP_USERNAME = "hadoop";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
index 030250101a2..024c1445e18 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreBaseProperties.java
@@ -73,6 +73,7 @@ public class AWSGlueMetaStoreBaseProperties {
public static AWSGlueMetaStoreBaseProperties of(Map<String, String>
properties) {
AWSGlueMetaStoreBaseProperties propertiesObj = new
AWSGlueMetaStoreBaseProperties();
ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj,
properties);
+ propertiesObj.checkAndInit();
return propertiesObj;
}
@@ -102,7 +103,7 @@ public class AWSGlueMetaStoreBaseProperties {
.require(glueEndpoint, "glue.endpoint or aws.endpoint or
aws.glue.endpoint is required");
}
- public void checkAndInit() {
+ private void checkAndInit() {
buildRules().validate();
Matcher matcher = ENDPOINT_PATTERN.matcher(glueEndpoint.toLowerCase());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
index ae0f98f61c2..8dcee8eb3ea 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AWSGlueMetaStoreProperties.java
@@ -42,7 +42,6 @@ public class AWSGlueMetaStoreProperties extends
MetastoreProperties {
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
- baseProperties.checkAndInit();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
index ca78929f405..fc68e74efa4 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AliyunDLFBaseProperties.java
@@ -43,6 +43,7 @@ public class AliyunDLFBaseProperties {
protected String dlfSessionToken = "";
@ConnectorProperty(names = {"dlf.region"},
+ required = false,
description = "The region of the Aliyun DLF.")
protected String dlfRegion = "";
@@ -84,7 +85,7 @@ public class AliyunDLFBaseProperties {
.require(dlfSecretKey, "dlf.secret_key is required");
}
- public void checkAndInit() {
+ private void checkAndInit() {
buildRules().validate();
if (StringUtils.isBlank(dlfEndpoint) &&
StringUtils.isNotBlank(dlfRegion)) {
if (BooleanUtils.toBoolean(dlfAccessPublic)) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
index c4a7661ad51..c0ea817d156 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSBaseProperties.java
@@ -68,9 +68,15 @@ public class HMSBaseProperties {
@ConnectorProperty(names = {"hadoop.security.authentication"},
required = false,
- description = "The authentication type of HDFS. The default value
is 'none'.")
+ description = "The authentication type of HDFS. The default value
is 'simple'.")
private String hdfsAuthenticationType = "";
+ @ConnectorProperty(names = {"hive.metastore.username", "hadoop.username"},
+ required = false,
+ description = "The user name for the Hive Metastore service. "
+ + "If not set, it will use the 'hadoop'.")
+ private String hmsUserName;
+
@ConnectorProperty(names = {"hadoop.kerberos.principal"},
required = false,
description = "The principal of the kerberos authentication.")
@@ -98,6 +104,7 @@ public class HMSBaseProperties {
public static HMSBaseProperties of(Map<String, String> properties) {
HMSBaseProperties propertiesObj = new HMSBaseProperties(properties);
ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj,
properties);
+ propertiesObj.checkAndInit();
return propertiesObj;
}
@@ -137,6 +144,10 @@ public class HMSBaseProperties {
if (StringUtils.isNotBlank(hiveMetastoreServicePrincipal)) {
hiveConf.set("hive.metastore.kerberos.principal",
hiveMetastoreServicePrincipal);
}
+ if
(StringUtils.isNotBlank(origProps.get(AuthenticationConfig.HADOOP_SECURITY_AUTH_TO_LOCAL)))
{
+ hiveConf.set(AuthenticationConfig.HADOOP_SECURITY_AUTH_TO_LOCAL,
+
origProps.get(AuthenticationConfig.HADOOP_SECURITY_AUTH_TO_LOCAL));
+ }
if (this.hiveMetastoreAuthenticationType.equalsIgnoreCase("kerberos"))
{
hiveConf.set("hadoop.security.authentication", "kerberos");
KerberosAuthenticationConfig authenticationConfig = new
KerberosAuthenticationConfig(
@@ -169,12 +180,15 @@ public class HMSBaseProperties {
return
CatalogConfigFileUtils.loadHiveConfFromHiveConfDir(resourceConfig);
}
- public void initAndCheckParams() {
+ private void checkAndInit() {
buildRules().validate();
this.hiveConf = loadHiveConfFromFile(hiveConfResourcesConfig);
initUserHiveConfig(origProps);
userOverriddenHiveConfig.forEach(hiveConf::set);
hiveConf.set("hive.metastore.uris", hiveMetastoreUri);
+ if (StringUtils.isNotBlank(hmsUserName)) {
+ hiveConf.set(AuthenticationConfig.HADOOP_USER_NAME, hmsUserName);
+ }
HiveConf.setVar(hiveConf,
HiveConf.ConfVars.METASTORE_CLIENT_SOCKET_TIMEOUT,
String.valueOf(Config.hive_metastore_client_timeout_second));
initHadoopAuthenticator();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
index 1826c593a0d..477dba85ebf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSGlueMetaStoreProperties.java
@@ -86,7 +86,6 @@ public class HMSGlueMetaStoreProperties extends
AbstractHMSProperties {
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
baseProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
- baseProperties.checkAndInit();
initHiveConf();
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
index fa09fe6ddc5..570be49b647 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/HMSProperties.java
@@ -55,7 +55,6 @@ public class HMSProperties extends AbstractHMSProperties {
super.initNormalizeAndCheckProps();
initRefreshParams();
hmsBaseProperties = HMSBaseProperties.of(origProps);
- hmsBaseProperties.initAndCheckParams();
this.hiveConf = hmsBaseProperties.getHiveConf();
this.executionAuthenticator = new
HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());
}
@@ -66,6 +65,4 @@ public class HMSProperties extends AbstractHMSProperties {
this.hmsEventsBatchSizePerRpc = hmsEventisBatchSizePerRpcInput;
}
-
-
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
index 794897394bc..a544029732f 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergGlueMetaStoreProperties.java
@@ -54,9 +54,7 @@ public class IcebergGlueMetaStoreProperties extends
AbstractIcebergProperties {
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
glueProperties = AWSGlueMetaStoreBaseProperties.of(origProps);
- glueProperties.checkAndInit();
s3Properties = S3Properties.of(origProps);
- s3Properties.initNormalizeAndCheckProps();
}
@Override
@@ -64,10 +62,8 @@ public class IcebergGlueMetaStoreProperties extends
AbstractIcebergProperties {
Map<String, String> props = prepareBaseCatalogProps();
appendS3Props(props);
appendGlueProps(props);
-
props.put("client.region", glueProperties.glueRegion);
-
if (StringUtils.isNotBlank(warehouse)) {
props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
} else {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
index ee589ba76fc..9f3bdb3b686 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergHMSMetaStoreProperties.java
@@ -59,7 +59,6 @@ public class IcebergHMSMetaStoreProperties extends
AbstractIcebergProperties {
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
hmsBaseProperties = HMSBaseProperties.of(origProps);
- hmsBaseProperties.initAndCheckParams();
this.executionAuthenticator = new
HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
index 9cb84903663..4f40bc131e1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/IcebergS3TablesMetaStoreProperties.java
@@ -48,7 +48,6 @@ public class IcebergS3TablesMetaStoreProperties extends
AbstractIcebergPropertie
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
s3Properties = S3Properties.of(origProps);
- s3Properties.initNormalizeAndCheckProps();
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java
index adfa53eab94..9a74775b9b2 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonFileSystemMetaStoreProperties.java
@@ -39,7 +39,7 @@ public class PaimonFileSystemMetaStoreProperties extends
AbstractPaimonPropertie
@Override
public Catalog initializeCatalog(String catalogName,
List<StorageProperties> storagePropertiesList) {
buildCatalogOptions(storagePropertiesList);
- Configuration conf = new Configuration(false);
+ Configuration conf = new Configuration();
storagePropertiesList.forEach(storageProperties -> {
for (Map.Entry<String, String> entry :
storageProperties.getHadoopStorageConfig()) {
catalogOptions.set(entry.getKey(), entry.getValue());
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
index 7c2d2ee79a3..a7afa94a9e0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/PaimonHMSMetaStoreProperties.java
@@ -66,7 +66,6 @@ public class PaimonHMSMetaStoreProperties extends
AbstractPaimonProperties {
public void initNormalizeAndCheckProps() {
super.initNormalizeAndCheckProps();
hmsBaseProperties = HMSBaseProperties.of(origProps);
- hmsBaseProperties.initAndCheckParams();
this.executionAuthenticator = new
HadoopExecutionAuthenticator(hmsBaseProperties.getHmsAuthenticator());
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
index f8bbc6bc3e5..7985d81ea47 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/storage/S3Properties.java
@@ -129,6 +129,7 @@ public class S3Properties extends
AbstractS3CompatibleProperties {
public static S3Properties of(Map<String, String> properties) {
S3Properties propertiesObj = new S3Properties(properties);
ConnectorPropertiesUtils.bindConnectorProperties(propertiesObj,
properties);
+ propertiesObj.initNormalizeAndCheckProps();
return propertiesObj;
}
diff --git
a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
index c3c97f0ca0c..0ab112d1dfb 100644
---
a/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
+++
b/regression-test/suites/external_table_p0/kerberos/test_single_hive_kerberos.groovy
@@ -63,6 +63,10 @@ suite("test_single_hive_kerberos",
"p0,external,kerberos,external_docker,externa
"fs.defaultFS" = "hdfs://${externalEnvIp}:8520",
"hadoop.security.authentication" = "kerberos",
"hadoop.kerberos.principal"="presto-server/[email protected]",
+ "hadoop.security.auth_to_local" =
"RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
+
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
+
RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
+ DEFAULT",
"hadoop.kerberos.keytab" =
"${keytab_root_dir}/presto-server.keytab"
);
"""
diff --git
a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
index c86c5a9dfd7..3e8627a09c7 100644
---
a/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
+++
b/regression-test/suites/external_table_p0/kerberos/test_two_hive_kerberos.groovy
@@ -47,6 +47,10 @@ suite("test_two_hive_kerberos",
"p0,external,kerberos,external_docker,external_d
"hadoop.kerberos.principal"="hive/[email protected]",
"hadoop.kerberos.keytab" =
"${keytab_root_dir}/hive-presto-master.keytab",
"hive.metastore.sasl.enabled " = "true",
+ "hadoop.security.auth_to_local" =
"RULE:[2:\\\$1@\\\$0](.*@LABS.TERADATA.COM)s/@.*//
+
RULE:[2:\\\$1@\\\$0](.*@OTHERLABS.TERADATA.COM)s/@.*//
+
RULE:[2:\\\$1@\\\$0](.*@OTHERREALM.COM)s/@.*//
+ DEFAULT",
"hive.metastore.kerberos.principal" =
"hive/[email protected]"
);
"""
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
