This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit edfcf00d92982e4654b13366843d62b9125a5a72
Author: Yulei-Yang <yulei.yang0...@gmail.com>
AuthorDate: Sun Apr 16 18:11:57 2023 +0800
[feature](multicatalog) enable doris hive/iceberg catalog to read data on
tencent GooseFS (#18685)
---
.../java/org/apache/doris/common/FeConstants.java | 1 +
.../doris/planner/external/HiveScanProvider.java | 2 +
.../external/iceberg/IcebergScanProvider.java | 2 +
.../doris/broker/hdfs/FileSystemManager.java | 43 +++++++++++++++++++++-
4 files changed, 47 insertions(+), 1 deletion(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
index 1b05795220..8d032b604a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/FeConstants.java
@@ -77,6 +77,7 @@ public class FeConstants {
public static String FS_PREFIX_COS = "cos";
public static String FS_PREFIX_OBS = "obs";
public static String FS_PREFIX_OFS = "ofs";
+ public static String FS_PREFIX_GFS = "gfs";
public static String FS_PREFIX_JFS = "jfs";
public static String FS_PREFIX_HDFS = "hdfs";
public static String FS_PREFIX_FILE = "file";
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
index f538bf3c3e..ec1d9f6ebf 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
@@ -126,6 +126,8 @@ public class HiveScanProvider extends HMSTableScanProvider {
return TFileType.FILE_LOCAL;
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
return TFileType.FILE_BROKER;
+ } else if (location.startsWith(FeConstants.FS_PREFIX_GFS)) {
+ return TFileType.FILE_BROKER;
} else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {
return TFileType.FILE_BROKER;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
index 6c4fb85055..2ef69e009c 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/planner/external/iceberg/IcebergScanProvider.java
@@ -131,6 +131,8 @@ public class IcebergScanProvider extends QueryScanProvider {
return TFileType.FILE_LOCAL;
} else if (location.startsWith(FeConstants.FS_PREFIX_OFS)) {
return TFileType.FILE_BROKER;
+ } else if (location.startsWith(FeConstants.FS_PREFIX_GFS)) {
+ return TFileType.FILE_BROKER;
} else if (location.startsWith(FeConstants.FS_PREFIX_JFS)) {
return TFileType.FILE_BROKER;
}
diff --git
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
index f66872a10a..f6e27bd0ef 100644
---
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
+++
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
@@ -75,6 +75,7 @@ public class FileSystemManager {
private static final String BOS_SCHEME = "bos";
private static final String JFS_SCHEME = "jfs";
private static final String AFS_SCHEME = "afs";
+ private static final String GFS_SCHEME = "gfs";
private static final String USER_NAME_KEY = "username";
private static final String PASSWORD_KEY = "password";
@@ -221,7 +222,9 @@ public class FileSystemManager {
brokerFileSystem = getBOSFileSystem(path, properties);
} else if (scheme.equals(JFS_SCHEME)) {
brokerFileSystem = getJuiceFileSystem(path, properties);
- }else {
+ } else if (scheme.equals(GFS_SCHEME)) {
+ brokerFileSystem = getGooseFSFileSystem(path, properties);
+ } else {
throw new
BrokerException(TBrokerOperationStatusCode.INVALID_INPUT_FILE_PATH,
"invalid path. scheme is not supported");
}
@@ -973,6 +976,44 @@ public class FileSystemManager {
}
}
+ /**
+ * @param path
+ * @param properties
+ * @return
+ * @throws URISyntaxException
+ * @throws Exception
+ */
+ public BrokerFileSystem getGooseFSFileSystem(String path, Map<String,
String> properties) {
+ WildcardURI pathUri = new WildcardURI(path);
+ // endpoint is the server host, pathUri.getUri().getHost() is the
bucket
+ // we should use these two params as the host identity, because
FileSystem will cache both.
+ String host = GFS_SCHEME + "://" + pathUri.getAuthority();
+
+ String username = properties.getOrDefault(USER_NAME_KEY, "");
+ String password = properties.getOrDefault(PASSWORD_KEY, "");
+ String gfsUgi = username + "," + password;
+ FileSystemIdentity fileSystemIdentity = new FileSystemIdentity(host,
gfsUgi);
+ BrokerFileSystem brokerFileSystem =
updateCachedFileSystem(fileSystemIdentity, properties);
+ brokerFileSystem.getLock().lock();
+ try {
+ if (brokerFileSystem.getDFSFileSystem() == null) {
+ logger.info("create goosefs client: " + path);
+ Configuration conf = new Configuration();
+ for (Map.Entry<String, String> propElement :
properties.entrySet()) {
+ conf.set(propElement.getKey(), propElement.getValue());
+ }
+ FileSystem fileSystem = FileSystem.get(pathUri.getUri(), conf);
+ brokerFileSystem.setFileSystem(fileSystem);
+ }
+ return brokerFileSystem;
+ } catch (Exception e) {
+ logger.error("errors while connect to " + path, e);
+ throw new
BrokerException(TBrokerOperationStatusCode.NOT_AUTHORIZED, e);
+ } finally {
+ brokerFileSystem.getLock().unlock();
+ }
+ }
+
public List<TBrokerFileStatus> listPath(String path, boolean fileNameOnly,
Map<String, String> properties) {
List<TBrokerFileStatus> resultFileStatus = null;
WildcardURI pathUri = new WildcardURI(path);
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
