Author: rwhitcomb
Date: Thu Jan 29 18:01:14 2015
New Revision: 1655773
URL: http://svn.apache.org/r1655773
Log:
VFS-555: Add the ability to specify an alternate HDFS configuration file
to override any specific HDFS settings. One use for this is to set a
different value for the dfs.client.use.datanode.hostname property in order
to access HDFS files stored in an AWS installation (from outside their
firewall). There are other possible uses too.
There is now an "configName" property in HdfsFileSystemConfigBuilder which
can be set in the file system options in order to set an alternate config
file. This file can have any number of option settings which will add to
or override the default settings. In other words, this file is loaded
after the default one (which resides in the HDFS .jar file).
Modified:
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java
Modified:
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
URL:
http://svn.apache.org/viewvc/commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java?rev=1655773&r1=1655772&r2=1655773&view=diff
==============================================================================
---
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
(original)
+++
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystem.java
Thu Jan 29 18:01:14 2015
@@ -109,6 +109,11 @@ public class HdfsFileSystem extends Abst
final String hdfsUri = name.getRootURI();
final Configuration conf = new Configuration(true);
conf.set(FileSystem.FS_DEFAULT_NAME_KEY, hdfsUri);
+ String configName =
HdfsFileSystemConfigBuilder.getInstance().getConfigName(getFileSystemOptions());
+ if (configName != null) {
+ log.debug("Adding alternate configuration file: " +
configName);
+ conf.addResource(configName);
+ }
this.fs = null;
try
{
@@ -147,7 +152,7 @@ public class HdfsFileSystem extends Abst
file = new HdfsFileObject((AbstractFileName) name, this, fs,
filePath);
if (useCache)
{
- this.putFileToCache(file);
+ this.putFileToCache(file);
}
}
/**
Modified:
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java
URL:
http://svn.apache.org/viewvc/commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java?rev=1655773&r1=1655772&r2=1655773&view=diff
==============================================================================
---
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java
(original)
+++
commons/proper/vfs/trunk/core/src/main/java/org/apache/commons/vfs2/provider/hdfs/HdfsFileSystemConfigBuilder.java
Thu Jan 29 18:01:14 2015
@@ -18,6 +18,7 @@ package org.apache.commons.vfs2.provider
import org.apache.commons.vfs2.FileSystem;
import org.apache.commons.vfs2.FileSystemConfigBuilder;
+import org.apache.commons.vfs2.FileSystemOptions;
/**
* Configuration settings for the HdfsFileSystem.
@@ -28,6 +29,10 @@ public class HdfsFileSystemConfigBuilder
{
private static final HdfsFileSystemConfigBuilder BUILDER = new
HdfsFileSystemConfigBuilder();
+ private HdfsFileSystemConfigBuilder() {
+ super("hdfs.");
+ }
+
/**
* @return HdfsFileSystemConfigBuilder instance
*/
@@ -45,4 +50,18 @@ public class HdfsFileSystemConfigBuilder
return HdfsFileSystem.class;
}
+ private static final String CONFIG_NAME = "config.name";
+
+ public String getConfigName(final FileSystemOptions opts) {
+ return this.getString(opts, CONFIG_NAME);
+ }
+
+ /**
+ * Sets the name of an alternate configuration file to be loaded after the
defaults.
+ */
+ public void setConfigName(final FileSystemOptions opts, final String name)
{
+ this.setParam(opts, CONFIG_NAME, name);
+ }
+
+
}
