This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f6986e1673d [fix](broker) Broker already had baidu afs support but
can't really use, so i fix it (#38275)
f6986e1673d is described below
commit f6986e1673d8978919c631ade815a1e244f328ed
Author: sparrow <38098988+biohazard4...@users.noreply.github.com>
AuthorDate: Thu Jul 25 11:40:55 2024 +0800
[fix](broker) Broker already had baidu afs support but can't really use, so
i fix it (#38275)
## Proposed changes
Issue Number: close #xxx
Add afs to the scheme options so broker can recognize afs location, and
change default afs impl to pure java implementation.
---
.../src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
index beeff588bcb..ad1a0bb03cc 100644
---
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
+++
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
@@ -234,6 +234,8 @@ public class FileSystemManager {
brokerFileSystem = getGooseFSFileSystem(path, properties);
} else if (scheme.equals(GCS_SCHEME)) {
brokerFileSystem = getGCSFileSystem(path, properties);
+ } else if (scheme.equals(AFS_SCHEME)) {
+ brokerFileSystem = getAfsFileSystem(path, properties);
} else {
throw new
BrokerException(TBrokerOperationStatusCode.INVALID_INPUT_FILE_PATH,
"invalid path. scheme is not supported");
@@ -1008,7 +1010,7 @@ public class FileSystemManager {
conf.set(HADOOP_JOB_UGI, afsUgi);
conf.set(HADOOP_JOB_GROUP_NAME, group);
conf.set(FS_DEFAULT_NAME, host);
- conf.set(FS_AFS_IMPL, "org.apache.hadoop.fs.DFileSystem");
+ conf.set(FS_AFS_IMPL, properties.getOrDefault(FS_AFS_IMPL,
"org.apache.hadoop.fs.LiteFileSystem"));
conf.set(DFS_CLIENT_AUTH_METHOD,
properties.getOrDefault(DFS_CLIENT_AUTH_METHOD, "3"));
conf.set(DFS_AGENT_PORT,
properties.getOrDefault(DFS_AGENT_PORT, "20001"));
conf.set(DFS_RPC_TIMEOUT,
properties.getOrDefault(DFS_RPC_TIMEOUT, "300000"));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org
