This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 0e7b16394fd [fix](paimon) fix hadoop.username does not take effect in
paimon catalog (#31478)
0e7b16394fd is described below
commit 0e7b16394fd8baf2d92e00fcf8e21e7ef00b4452
Author: Mingyu Chen <[email protected]>
AuthorDate: Wed Feb 28 13:01:06 2024 +0800
[fix](paimon) fix hadoop.username does not take effect in paimon catalog
(#31478)
---
.../java/org/apache/doris/hudi/HudiJniScanner.java | 15 ++----
.../src/main/java/org/apache/doris/hudi/Utils.java | 22 +-------
.../authentication/AuthenticationConfig.java | 8 +--
.../common/security/authentication/HadoopUGI.java | 20 ++++++-
.../datasource/hive/HiveMetaStoreClientHelper.java | 33 ++----------
.../datasource/paimon/PaimonExternalCatalog.java | 63 ++++++++++++++--------
.../paimon/PaimonFileExternalCatalog.java | 1 +
.../apache/doris/fs/remote/dfs/DFSFileSystem.java | 21 +++-----
.../ExternalFileTableValuedFunction.java | 4 +-
9 files changed, 81 insertions(+), 106 deletions(-)
diff --git
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
index f22a69255a9..932b53b9a7c 100644
---
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
+++
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java
@@ -27,7 +27,6 @@ import
org.apache.doris.common.security.authentication.HadoopUGI;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.avro.generic.GenericDatumReader;
import org.apache.avro.util.WeakIdentityHashMap;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.log4j.Logger;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.sources.Filter;
@@ -36,7 +35,6 @@ import scala.collection.Iterator;
import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.Field;
-import java.security.PrivilegedExceptionAction;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -63,7 +61,6 @@ public class HudiJniScanner extends JniScanner {
private final HoodieSplit split;
private final ScanPredicate[] predicates;
private final ClassLoader classLoader;
- private final UserGroupInformation ugi;
private long getRecordReaderTimeNs = 0;
private Iterator<InternalRow> recordIterator;
@@ -140,7 +137,6 @@ public class HudiJniScanner extends JniScanner {
predicates = new ScanPredicate[0];
}
}
- ugi =
HadoopUGI.loginWithUGI(AuthenticationConfig.getKerberosConfig(split.hadoopConf()));
} catch (Exception e) {
LOG.error("Failed to initialize hudi scanner, split params:\n" +
debugString, e);
throw e;
@@ -178,14 +174,9 @@ public class HudiJniScanner extends JniScanner {
cleanResolverLock.readLock().lock();
try {
lastUpdateTime.set(System.currentTimeMillis());
- if (ugi != null) {
- recordIterator = ugi.doAs(
- (PrivilegedExceptionAction<Iterator<InternalRow>>)
() -> new MORSnapshotSplitReader(
- split).buildScanIterator(new Filter[0]));
- } else {
- recordIterator = new MORSnapshotSplitReader(split)
- .buildScanIterator(new Filter[0]);
- }
+ recordIterator = HadoopUGI.ugiDoAs(
+
AuthenticationConfig.getKerberosConfig(split.hadoopConf()), () -> new
MORSnapshotSplitReader(
+ split).buildScanIterator(new Filter[0]));
if (AVRO_RESOLVER_CACHE != null && AVRO_RESOLVER_CACHE.get()
!= null) {
cachedResolvers.computeIfAbsent(Thread.currentThread().getId(),
threadId -> AVRO_RESOLVER_CACHE.get());
diff --git
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
index be5628d2ce4..03085b12f2b 100644
---
a/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
+++
b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java
@@ -22,7 +22,6 @@ import
org.apache.doris.common.security.authentication.HadoopUGI;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import sun.management.VMManagement;
@@ -34,12 +33,10 @@ import java.lang.management.ManagementFactory;
import java.lang.management.RuntimeMXBean;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
-import java.security.PrivilegedExceptionAction;
import java.util.LinkedList;
import java.util.List;
public class Utils {
-
public static long getCurrentProcId() {
try {
RuntimeMXBean mxbean = ManagementFactory.getRuntimeMXBean();
@@ -88,22 +85,7 @@ public class Utils {
}
public static HoodieTableMetaClient getMetaClient(Configuration conf,
String basePath) {
- UserGroupInformation ugi =
HadoopUGI.loginWithUGI(AuthenticationConfig.getKerberosConfig(conf));
- HoodieTableMetaClient metaClient;
- if (ugi != null) {
- try {
- ugi.checkTGTAndReloginFromKeytab();
- metaClient = ugi.doAs(
- (PrivilegedExceptionAction<HoodieTableMetaClient>) ()
-> HoodieTableMetaClient.builder()
- .setConf(conf).setBasePath(basePath).build());
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (InterruptedException e) {
- throw new RuntimeException("Cannot get hudi client.", e);
- }
- } else {
- metaClient =
HoodieTableMetaClient.builder().setConf(conf).setBasePath(basePath).build();
- }
- return metaClient;
+ return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
() -> HoodieTableMetaClient.builder()
+ .setConf(conf).setBasePath(basePath).build());
}
}
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
index b3cb69f7004..315cb901e2b 100644
---
a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
+++
b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/AuthenticationConfig.java
@@ -17,12 +17,9 @@
package org.apache.doris.common.security.authentication;
-import lombok.Data;
import org.apache.hadoop.conf.Configuration;
-@Data
public abstract class AuthenticationConfig {
-
public static String HADOOP_USER_NAME = "hadoop.username";
public static String HADOOP_SECURITY_AUTHENTICATION =
"hadoop.security.authentication";
public static String HADOOP_KERBEROS_PRINCIPAL =
"hadoop.kerberos.principal";
@@ -31,7 +28,10 @@ public abstract class AuthenticationConfig {
public static String HIVE_KERBEROS_PRINCIPAL =
"hive.metastore.kerberos.principal";
public static String HIVE_KERBEROS_KEYTAB =
"hive.metastore.kerberos.keytab.file";
- private boolean isValid;
+ /**
+ * @return true if the config is valid, otherwise false.
+ */
+ public abstract boolean isValid();
/**
* get kerberos config from hadoop conf
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java
b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java
index 5fb8f4fdab1..3cd419ff706 100644
---
a/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java
+++
b/fe/fe-common/src/main/java/org/apache/doris/common/security/authentication/HadoopUGI.java
@@ -24,6 +24,7 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
+import java.security.PrivilegedExceptionAction;
public class HadoopUGI {
private static final Logger LOG = LogManager.getLogger(HadoopUGI.class);
@@ -33,7 +34,10 @@ public class HadoopUGI {
* @param config auth config
* @return ugi
*/
- public static UserGroupInformation loginWithUGI(AuthenticationConfig
config) {
+ private static UserGroupInformation loginWithUGI(AuthenticationConfig
config) {
+ if (config == null || !config.isValid()) {
+ return null;
+ }
UserGroupInformation ugi;
if (config instanceof KerberosAuthenticationConfig) {
KerberosAuthenticationConfig krbConfig =
(KerberosAuthenticationConfig) config;
@@ -96,4 +100,18 @@ public class HadoopUGI {
}
}
}
+
+ public static <T> T ugiDoAs(AuthenticationConfig authConf,
PrivilegedExceptionAction<T> action) {
+ UserGroupInformation ugi = HadoopUGI.loginWithUGI(authConf);
+ try {
+ if (ugi != null) {
+ ugi.checkTGTAndReloginFromKeytab();
+ return ugi.doAs(action);
+ } else {
+ return action.run();
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e.getMessage(), e);
+ }
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
index 037fbe02d68..23c83a11146 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreClientHelper.java
@@ -71,7 +71,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hudi.avro.HoodieAvroUtils;
import org.apache.hudi.common.table.HoodieTableMetaClient;
import org.apache.hudi.common.table.TableSchemaResolver;
@@ -79,7 +78,6 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import shade.doris.hive.org.apache.thrift.TException;
-import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.time.LocalDateTime;
import java.time.ZoneId;
@@ -828,39 +826,14 @@ public class HiveMetaStoreClientHelper {
AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL,
AuthenticationConfig.HADOOP_KERBEROS_KEYTAB);
}
- UserGroupInformation ugi = HadoopUGI.loginWithUGI(krbConfig);
- try {
- if (ugi != null) {
- ugi.checkTGTAndReloginFromKeytab();
- return ugi.doAs(action);
- } else {
- return action.run();
- }
- } catch (Exception e) {
- throw new RuntimeException(e.getMessage(), e);
- }
+ return HadoopUGI.ugiDoAs(krbConfig, action);
}
public static HoodieTableMetaClient getHudiClient(HMSExternalTable table) {
String hudiBasePath = table.getRemoteTable().getSd().getLocation();
-
Configuration conf = getConfiguration(table);
- UserGroupInformation ugi =
HadoopUGI.loginWithUGI(AuthenticationConfig.getKerberosConfig(conf));
- HoodieTableMetaClient metaClient;
- if (ugi != null) {
- try {
- metaClient = ugi.doAs(
- (PrivilegedExceptionAction<HoodieTableMetaClient>) ()
-> HoodieTableMetaClient.builder()
-
.setConf(conf).setBasePath(hudiBasePath).build());
- } catch (IOException e) {
- throw new RuntimeException(e);
- } catch (InterruptedException e) {
- throw new RuntimeException("Cannot get hudi client.", e);
- }
- } else {
- metaClient =
HoodieTableMetaClient.builder().setConf(conf).setBasePath(hudiBasePath).build();
- }
- return metaClient;
+ return HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf),
+ () ->
HoodieTableMetaClient.builder().setConf(conf).setBasePath(hudiBasePath).build());
}
public static Configuration getConfiguration(HMSExternalTable table) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
index 2f7aaa18eb8..ec0a0586937 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java
@@ -18,6 +18,8 @@
package org.apache.doris.datasource.paimon;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.security.authentication.AuthenticationConfig;
+import org.apache.doris.common.security.authentication.HadoopUGI;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
@@ -25,12 +27,15 @@ import
org.apache.doris.datasource.property.constants.PaimonProperties;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Maps;
+import org.apache.hadoop.conf.Configuration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.paimon.catalog.Catalog;
import org.apache.paimon.catalog.CatalogContext;
import org.apache.paimon.catalog.CatalogFactory;
+import org.apache.paimon.catalog.FileSystemCatalog;
import org.apache.paimon.catalog.Identifier;
+import org.apache.paimon.hive.HiveCatalog;
import org.apache.paimon.options.Options;
import java.util.ArrayList;
@@ -44,6 +49,7 @@ public abstract class PaimonExternalCatalog extends
ExternalCatalog {
public static final String PAIMON_HMS = "hms";
protected String catalogType;
protected Catalog catalog;
+ protected AuthenticationConfig authConf;
private static final List<String> REQUIRED_PROPERTIES = ImmutableList.of(
PaimonProperties.WAREHOUSE
@@ -54,13 +60,20 @@ public abstract class PaimonExternalCatalog extends
ExternalCatalog {
}
@Override
- protected void init() {
- super.init();
- }
-
- public Catalog getCatalog() {
- makeSureInitialized();
- return catalog;
+ protected void initLocalObjectsImpl() {
+ Configuration conf = new Configuration();
+ for (Map.Entry<String, String> propEntry :
this.catalogProperty.getHadoopProperties().entrySet()) {
+ conf.set(propEntry.getKey(), propEntry.getValue());
+ }
+ if (catalog instanceof FileSystemCatalog) {
+ authConf = AuthenticationConfig.getKerberosConfig(conf,
+ AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL,
+ AuthenticationConfig.HADOOP_KERBEROS_KEYTAB);
+ } else if (catalog instanceof HiveCatalog) {
+ authConf = AuthenticationConfig.getKerberosConfig(conf,
+ AuthenticationConfig.HIVE_KERBEROS_PRINCIPAL,
+ AuthenticationConfig.HIVE_KERBEROS_KEYTAB);
+ }
}
public String getCatalogType() {
@@ -69,36 +82,40 @@ public abstract class PaimonExternalCatalog extends
ExternalCatalog {
}
protected List<String> listDatabaseNames() {
- return new ArrayList<>(catalog.listDatabases());
+ return HadoopUGI.ugiDoAs(authConf, () -> new
ArrayList<>(catalog.listDatabases()));
}
@Override
public boolean tableExist(SessionContext ctx, String dbName, String
tblName) {
makeSureInitialized();
- return catalog.tableExists(Identifier.create(dbName, tblName));
+ return HadoopUGI.ugiDoAs(authConf, () ->
catalog.tableExists(Identifier.create(dbName, tblName)));
}
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
makeSureInitialized();
- List<String> tableNames = null;
- try {
- tableNames = catalog.listTables(dbName);
- } catch (Catalog.DatabaseNotExistException e) {
- LOG.warn("DatabaseNotExistException", e);
- }
- return tableNames;
+ return HadoopUGI.ugiDoAs(authConf, () -> {
+ List<String> tableNames = null;
+ try {
+ tableNames = catalog.listTables(dbName);
+ } catch (Catalog.DatabaseNotExistException e) {
+ LOG.warn("DatabaseNotExistException", e);
+ }
+ return tableNames;
+ });
}
public org.apache.paimon.table.Table getPaimonTable(String dbName, String
tblName) {
makeSureInitialized();
- org.apache.paimon.table.Table table = null;
- try {
- table = catalog.getTable(Identifier.create(dbName, tblName));
- } catch (Catalog.TableNotExistException e) {
- LOG.warn("TableNotExistException", e);
- }
- return table;
+ return HadoopUGI.ugiDoAs(authConf, () -> {
+ org.apache.paimon.table.Table table = null;
+ try {
+ table = catalog.getTable(Identifier.create(dbName, tblName));
+ } catch (Catalog.TableNotExistException e) {
+ LOG.warn("TableNotExistException", e);
+ }
+ return table;
+ });
}
protected String getPaimonCatalogType(String catalogType) {
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
index 62b29c98f3a..f2f52c3d116 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonFileExternalCatalog.java
@@ -42,6 +42,7 @@ public class PaimonFileExternalCatalog extends
PaimonExternalCatalog {
protected void initLocalObjectsImpl() {
catalogType = PAIMON_FILESYSTEM;
catalog = createCatalog();
+ super.initLocalObjectsImpl();
}
@Override
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
index f28e2eb66a6..e27e27ddbff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/fs/remote/dfs/DFSFileSystem.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -51,7 +50,6 @@ import java.nio.ByteBuffer;
import java.nio.file.FileVisitOption;
import java.nio.file.Files;
import java.nio.file.Paths;
-import java.security.PrivilegedAction;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -82,18 +80,13 @@ public class DFSFileSystem extends RemoteFileSystem {
conf.set(propEntry.getKey(), propEntry.getValue());
}
- UserGroupInformation ugi =
HadoopUGI.loginWithUGI(AuthenticationConfig.getKerberosConfig(conf));
- try {
- dfsFileSystem = ugi.doAs((PrivilegedAction<FileSystem>) () -> {
- try {
- return FileSystem.get(new Path(remotePath).toUri(), conf);
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- });
- } catch (SecurityException e) {
- throw new UserException(e);
- }
+ dfsFileSystem =
HadoopUGI.ugiDoAs(AuthenticationConfig.getKerberosConfig(conf), () -> {
+ try {
+ return FileSystem.get(new Path(remotePath).toUri(), conf);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ });
Preconditions.checkNotNull(dfsFileSystem);
operations = new HDFSFileOperations(dfsFileSystem);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
index 2a6e15dba43..451b37d1311 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java
@@ -474,8 +474,8 @@ public abstract class ExternalFileTableValuedFunction
extends TableValuedFunctio
if (getTFileType() == TFileType.FILE_HDFS) {
THdfsParams tHdfsParams =
HdfsResource.generateHdfsParam(locationProperties);
- String fsNmae =
getLocationProperties().get(HdfsResource.HADOOP_FS_NAME);
- tHdfsParams.setFsName(fsNmae);
+ String fsName =
getLocationProperties().get(HdfsResource.HADOOP_FS_NAME);
+ tHdfsParams.setFsName(fsName);
fileScanRangeParams.setHdfsParams(tHdfsParams);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
