[incubator-doris] branch master updated: Fix 'Filesystem closed' in broker load (#3216)

[kangkaisen]

This is an automated email from the ASF dual-hosted git repository.

kangkaisen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git


The following commit(s) were added to refs/heads/master by this push:
     new 4a5164a  Fix 'Filesystem closed' in broker load (#3216)
4a5164a is described below

commit 4a5164ab9d1a4b0ca8e9b8a08f876f22a4a2c500
Author: frwrdt <38685812+frw...@users.noreply.github.com>
AuthorDate: Sat Mar 28 09:14:45 2020 +0800

    Fix 'Filesystem closed' in broker load (#3216)
---
 .../main/java/org/apache/doris/broker/hdfs/FileSystemManager.java  | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git 
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
 
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
index 3826f21..e801340 100644
--- 
a/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
+++ 
b/fs_brokers/apache_hdfs_broker/src/main/java/org/apache/doris/broker/hdfs/FileSystemManager.java
@@ -84,11 +84,16 @@ public class FileSystemManager {
     private static final String DEFAULT_DFS_CLIENT_FAILOVER_PROXY_PROVIDER =
             
"org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider";
     private static final String FS_DEFAULTFS_KEY = "fs.defaultFS";
+    // If this property is not set to "true", FileSystem instance will be 
returned from cache
+    // which is not thread-safe and may cause 'Filesystem closed' exception 
when it is closed by other thread.
+    private static final String FS_HDFS_IMPL_DISABLE_CACHE = 
"fs.hdfs.impl.disable.cache";
 
     // arguments for s3a
     private static final String FS_S3A_ACCESS_KEY = "fs.s3a.access.key";
     private static final String FS_S3A_SECRET_KEY = "fs.s3a.secret.key";
     private static final String FS_S3A_ENDPOINT = "fs.s3a.endpoint";
+    // This property is used like 'fs.hdfs.impl.disable.cache'
+    private static final String FS_S3A_IMPL_DISABLE_CACHE = 
"fs.s3a.impl.disable.cache";
 
     private ScheduledExecutorService handleManagementPool = 
Executors.newScheduledThreadPool(2);
     
@@ -332,6 +337,7 @@ public class FileSystemManager {
                     }
                 }
 
+                conf.set(FS_HDFS_IMPL_DISABLE_CACHE, "true");
                 FileSystem dfsFileSystem = FileSystem.get(pathUri.getUri(), 
conf);
                 fileSystem.setFileSystem(dfsFileSystem);
             }
@@ -383,6 +389,7 @@ public class FileSystemManager {
                 conf.set(FS_S3A_ACCESS_KEY, accessKey);
                 conf.set(FS_S3A_SECRET_KEY, secretKey);
                 conf.set(FS_S3A_ENDPOINT, endpoint);
+                conf.set(FS_S3A_IMPL_DISABLE_CACHE, "true");
                 FileSystem s3AFileSystem = FileSystem.get(pathUri.getUri(), 
conf);
                 fileSystem.setFileSystem(s3AFileSystem);
             }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org
For additional commands, e-mail: commits-h...@doris.apache.org