Repository: accumulo Updated Branches: refs/heads/1.6.0-SNAPSHOT ef5dc4a1f -> cb2f4b580 refs/heads/master 866422d27 -> 2dbc14fa9
http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java index 205a793..ad29c19 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/fs/FileTypeTest.java @@ -27,28 +27,50 @@ import org.junit.Test; public class FileTypeTest { @Test public void testVolumeExtraction() { - Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:///a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:/a/accumulo"), FileType.TABLE.getVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf"))); + // Having an 'accumulo' directory is not a requirement + Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:/a/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:///a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:/a"), FileType.TABLE.getVolume(new Path("file:///a/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/accumulo/wal/1.2.3.4/aaa-bbb-ccc-ddd"))); + // Having an 'accumulo' directory is not a requirement + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:/a/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("file:///a/tables/2b/t-001/C00.rf"))); + + Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:/accumulo"), FileType.TABLE.getVolume(new Path("file:///accumulo/tables/2b/t-001/C00.rf"))); + + // Having an 'accumulo' directory is not a requirement + Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("file:/"), FileType.TABLE.getVolume(new Path("file:///tables/2b/t-001/C00.rf"))); + + Assert.assertEquals(new Path("file:/a"), FileType.WAL.getVolume(new Path("file:/a/wal/1.2.3.4/aaa-bbb-ccc-ddd"))); Assert.assertNull(FileType.WAL.getVolume(new Path("1.2.3.4/aaa-bbb-ccc-ddd"))); Assert.assertNull(FileType.TABLE.getVolume(new Path("../2b/t-001/C00.rf"))); Assert.assertNull(FileType.TABLE.getVolume(new Path("/t-001/C00.rf"))); - Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("hdfs://nn1/a/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("hdfs://nn1/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("hdfs://nn1/a/accumulo"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf"))); + + // Having an 'accumulo' directory is not a requirement + Assert.assertEquals(new Path("hdfs://nn1/"), FileType.TABLE.getVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("hdfs://nn1/a"), FileType.TABLE.getVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf"))); + + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/accumulo/tables/2b/t-001/C00.rf"))); - Assert.assertEquals(new Path("accumulo/tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/accumulo/tables/2b/t-001/C00.rf"))); + // Having an 'accumulo' directory is not a requirement + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/tables/2b/t-001/C00.rf"))); + Assert.assertEquals(new Path("tables/2b/t-001/C00.rf"), FileType.TABLE.removeVolume(new Path("hdfs://nn1/a/tables/2b/t-001/C00.rf"))); } } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java index c85be45..3b905c9 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java @@ -43,9 +43,9 @@ public class VolumeUtilTest { @Test public void testSwitchVolume() { List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/"), new Path("viewfs:/b"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b/accumulo"))); Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf", VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); @@ -57,9 +57,9 @@ public class VolumeUtilTest { Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); replacements.clear(); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/"), new Path("viewfs:/b"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b/accumulo"))); Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf", VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); @@ -73,6 +73,70 @@ public class VolumeUtilTest { } @Test + public void testSwitchVolumesDifferentSourceDepths() { + List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b"))); + + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + + replacements.clear(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b"))); + + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + } + + @Test + public void testSwitchVolumesDifferentTargetDepths() { + List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/path3"))); + + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + + replacements.clear(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/path3"))); + + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + } + + @Test public void testSame() throws Exception { FileSystem fs = FileSystem.getLocal(new Configuration()); @@ -133,6 +197,19 @@ public class VolumeUtilTest { } + @Test + public void testRootTableReplacement() throws IOException { + List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); + replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8"))); + replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9"))); + + FileType ft = FileType.TABLE; + + Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet", + VolumeUtil.switchVolume("file:/foo/v1/tables/+r/root_tablet", + ft, replacements)); + } + private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException { FSDataOutputStream out = fs.create(new Path(dir, filename)); try { http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java ---------------------------------------------------------------------- diff --git a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java index 4e2a878..1e9c1dd 100644 --- a/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java +++ b/server/gc/src/test/java/org/apache/accumulo/gc/GarbageCollectionTest.java @@ -335,6 +335,71 @@ public class GarbageCollectionTest { assertRemoved(gce); } + + @Test + public void testCustomDirectories() throws Exception { + TestGCE gce = new TestGCE(); + + gce.candidates.add("/4/t-0"); + gce.candidates.add("/4/t-0/F002.rf"); + gce.candidates.add("hdfs://foo.com:6000/user/foo/tables/5/t-0"); + gce.candidates.add("/6/t-0"); + gce.candidates.add("hdfs://foo:6000/user/foo/tables/7/t-0/"); + gce.candidates.add("/8/t-0"); + gce.candidates.add("hdfs://foo:6000/user/foo/tables/9/t-0"); + gce.candidates.add("/a/t-0"); + gce.candidates.add("hdfs://foo:6000/user/foo/tables/b/t-0"); + gce.candidates.add("/c/t-0"); + gce.candidates.add("hdfs://foo:6000/user/foo/tables/d/t-0"); + + gce.addDirReference("4", null, "/t-0"); + gce.addDirReference("5", null, "/t-0"); + gce.addDirReference("6", null, "hdfs://foo.com:6000/user/foo/tables/6/t-0"); + gce.addDirReference("7", null, "hdfs://foo.com:6000/user/foo/tables/7/t-0"); + + gce.addFileReference("8", "m", "/t-0/F00.rf"); + gce.addFileReference("9", "m", "/t-0/F00.rf"); + + gce.addFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf"); + gce.addFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf"); + + gce.addFileReference("e", "m", "../c/t-0/F00.rf"); + gce.addFileReference("f", "m", "../d/t-0/F00.rf"); + + GarbageCollectionAlgorithm gca = new GarbageCollectionAlgorithm(); + + // A directory reference does not preclude a candidate file beneath that directory from deletion + gca.collect(gce); + assertRemoved(gce, "/4/t-0/F002.rf"); + + // Removing the dir reference for a table will delete all tablet directories + gce.removeDirReference("5", null); + gca.collect(gce); + assertRemoved(gce, "hdfs://foo.com:6000/user/foo/tables/5/t-0"); + + gce.removeDirReference("4", null); + gca.collect(gce); + assertRemoved(gce, "/4/t-0"); + + gce.removeDirReference("6", null); + gce.removeDirReference("7", null); + gca.collect(gce); + assertRemoved(gce, "/6/t-0", "hdfs://foo:6000/user/foo/tables/7/t-0/"); + + gce.removeFileReference("8", "m", "/t-0/F00.rf"); + gce.removeFileReference("9", "m", "/t-0/F00.rf"); + gce.removeFileReference("a", "m", "hdfs://foo.com:6000/user/foo/tables/a/t-0/F00.rf"); + gce.removeFileReference("b", "m", "hdfs://foo.com:6000/user/foo/tables/b/t-0/F00.rf"); + gce.removeFileReference("e", "m", "../c/t-0/F00.rf"); + gce.removeFileReference("f", "m", "../d/t-0/F00.rf"); + gca.collect(gce); + assertRemoved(gce, "/8/t-0", "hdfs://foo:6000/user/foo/tables/9/t-0", "/a/t-0", "hdfs://foo:6000/user/foo/tables/b/t-0", "/c/t-0", + "hdfs://foo:6000/user/foo/tables/d/t-0"); + + gca.collect(gce); + assertRemoved(gce); + } + private void badRefTest(String ref) { TestGCE gce = new TestGCE(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java index 36bbb53..eeb9b16 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ExportTable.java @@ -152,7 +152,7 @@ class WriteExportFiles extends MasterRepo { public static void exportTable(VolumeManager fs, Connector conn, String tableName, String tableID, String exportDir) throws Exception { fs.mkdirs(new Path(exportDir)); - Path exportMetaFilePath = fs.getFileSystemByPath(new Path(exportDir)).makeQualified(new Path(exportDir, Constants.EXPORT_FILE)); + Path exportMetaFilePath = fs.getVolumeByPath(new Path(exportDir)).getFileSystem().makeQualified(new Path(exportDir, Constants.EXPORT_FILE)); FSDataOutputStream fileOut = fs.create(exportMetaFilePath, false); ZipOutputStream zipOut = new ZipOutputStream(fileOut); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java ---------------------------------------------------------------------- diff --git a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java index 7e84c55..cd59b78 100644 --- a/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java +++ b/server/master/src/main/java/org/apache/accumulo/master/tableOps/ImportTable.java @@ -427,7 +427,7 @@ class ImportPopulateZookeeper extends MasterRepo { Path path = new Path(tableInfo.exportDir, Constants.EXPORT_FILE); try { - FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); return TableOperationsImpl.getExportedProps(ns, path); } catch (IOException ioe) { throw new ThriftTableOperationException(tableInfo.tableId, tableInfo.tableName, TableOperation.IMPORT, TableOperationExceptionType.OTHER, http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java ---------------------------------------------------------------------- diff --git a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java index 942f866..31e63ed 100644 --- a/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java +++ b/server/monitor/src/main/java/org/apache/accumulo/monitor/servlets/DefaultServlet.java @@ -33,11 +33,11 @@ import javax.servlet.http.HttpServletResponse; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.conf.DefaultConfiguration; -import org.apache.accumulo.core.file.VolumeConfiguration; import org.apache.accumulo.core.master.thrift.MasterMonitorInfo; import org.apache.accumulo.core.util.Duration; import org.apache.accumulo.core.util.NumUtil; import org.apache.accumulo.core.util.Pair; +import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.accumulo.monitor.Monitor; import org.apache.accumulo.monitor.ZooKeeperStatus; import org.apache.accumulo.monitor.ZooKeeperStatus.ZooKeeperState; @@ -268,9 +268,9 @@ public class DefaultServlet extends BasicServlet { long totalHdfsBytesUsed = 0l; try { - for (String baseDir : VolumeConfiguration.getConfiguredBaseDirs(ServerConfiguration.getSiteConfiguration())) { + for (String baseDir : VolumeConfiguration.getVolumeUris(ServerConfiguration.getSiteConfiguration())) { final Path basePath = new Path(baseDir); - final FileSystem fs = vm.getFileSystemByPath(basePath); + final FileSystem fs = vm.getVolumeByPath(basePath).getFileSystem(); try { // Calculate the amount of space used by Accumulo on the FileSystem http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java index e9f1083..6b31af1 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/BulkFailedCopyProcessor.java @@ -20,6 +20,7 @@ import java.io.IOException; import org.apache.accumulo.core.Constants; import org.apache.accumulo.core.util.CachedConfiguration; +import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.trace.TraceFileSystem; import org.apache.accumulo.server.zookeeper.DistributedWorkQueue.Processor; @@ -32,35 +33,35 @@ import org.apache.log4j.Logger; * Copy failed bulk imports. */ public class BulkFailedCopyProcessor implements Processor { - + private static final Logger log = Logger.getLogger(BulkFailedCopyProcessor.class); - + @Override public Processor newProcessor() { return new BulkFailedCopyProcessor(); } - + @Override public void process(String workID, byte[] data) { - + String paths[] = new String(data, Constants.UTF8).split(","); - + Path orig = new Path(paths[0]); Path dest = new Path(paths[1]); Path tmp = new Path(dest.getParent(), dest.getName() + ".tmp"); - + try { - FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(), - ServerConfiguration.getSiteConfiguration())); - + FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(), + ServerConfiguration.getSiteConfiguration()).getFileSystem()); + FileUtil.copy(fs, orig, fs, tmp, false, true, CachedConfiguration.getInstance()); fs.rename(tmp, dest); log.debug("copied " + orig + " to " + dest); } catch (IOException ex) { try { - FileSystem fs = TraceFileSystem.wrap(org.apache.accumulo.core.file.VolumeConfiguration.getDefaultFilesystem(CachedConfiguration.getInstance(), - ServerConfiguration.getSiteConfiguration())); - + FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(CachedConfiguration.getInstance(), + ServerConfiguration.getSiteConfiguration()).getFileSystem()); + fs.create(dest).close(); log.warn(" marked " + dest + " failed", ex); } catch (IOException e) { @@ -69,5 +70,5 @@ public class BulkFailedCopyProcessor implements Processor { } } - + } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java index 151db6e..822171c 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Compactor.java @@ -336,7 +336,7 @@ public class Compactor implements Callable<CompactionStats> { thread = Thread.currentThread(); try { FileOperations fileFactory = FileOperations.getInstance(); - FileSystem ns = this.fs.getFileSystemByPath(outputFile.path()); + FileSystem ns = this.fs.getVolumeByPath(outputFile.path()).getFileSystem(); mfw = fileFactory.openWriter(outputFile.path().toString(), ns, ns.getConf(), acuTableConf); Map<String,Set<ByteSequence>> lGroups; @@ -421,7 +421,7 @@ public class Compactor implements Callable<CompactionStats> { try { FileOperations fileFactory = FileOperations.getInstance(); - FileSystem fs = this.fs.getFileSystemByPath(mapFile.path()); + FileSystem fs = this.fs.getVolumeByPath(mapFile.path()).getFileSystem(); FileSKVIterator reader; reader = fileFactory.openReader(mapFile.path().toString(), false, fs, conf, acuTableConf); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java index bb95532..8bf2517 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/FileManager.java @@ -309,7 +309,7 @@ public class FileManager { if (!file.contains(":")) throw new IllegalArgumentException("Expected uri, got : " + file); Path path = new Path(file); - FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); //log.debug("Opening "+file + " path " + path); FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), false, ns, ns.getConf(), conf.getTableConfiguration(table.toString()), dataCache, indexCache); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java index cc4b68d..3fe60b7 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/Tablet.java @@ -1260,7 +1260,7 @@ public class Tablet { long rtime = Long.MIN_VALUE; for (FileRef ref : datafiles.keySet()) { Path path = ref.path(); - FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); FileSKVIterator reader = FileOperations.getInstance().openReader(path.toString(), true, ns, ns.getConf(), tabletServer.getTableConfiguration(extent)); long maxTime = -1; try { @@ -2975,7 +2975,7 @@ public class Tablet { FileOperations fileFactory = FileOperations.getInstance(); for (Entry<FileRef,DataFileValue> entry : allFiles.entrySet()) { FileRef file = entry.getKey(); - FileSystem ns = fs.getFileSystemByPath(file.path()); + FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem(); FileSKVIterator openReader = fileFactory.openReader(file.path().toString(), true, ns, ns.getConf(), this.getTableConfiguration()); try { Key first = openReader.getFirstKey(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java index 475621b..6d73125 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/TabletServer.java @@ -969,7 +969,7 @@ public class TabletServer extends AbstractMetricsImpl implements org.apache.accu Map<FileRef,MapFileInfo> fileRefMap = new HashMap<FileRef,MapFileInfo>(); for (Entry<String,MapFileInfo> mapping : fileMap.entrySet()) { Path path = new Path(mapping.getKey()); - FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); path = ns.makeQualified(path); fileRefMap.put(new FileRef(path.toString(), path), mapping.getValue()); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java index 3bbb476..900600f 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/compaction/MajorCompactionRequest.java @@ -81,7 +81,7 @@ public class MajorCompactionRequest implements Cloneable { // @TODO verify the file isn't some random file in HDFS // @TODO ensure these files are always closed? FileOperations fileFactory = FileOperations.getInstance(); - FileSystem ns = volumeManager.getFileSystemByPath(ref.path()); + FileSystem ns = volumeManager.getVolumeByPath(ref.path()).getFileSystem(); FileSKVIterator openReader = fileFactory.openReader(ref.path().toString(), true, ns, ns.getConf(), tableConfig); return openReader; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java index 8f783c3..bb8e3c7 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/LogSorter.java @@ -160,7 +160,7 @@ public class LogSorter { private void writeBuffer(String destPath, ArrayList<Pair<LogFileKey,LogFileValue>> buffer, int part) throws IOException { Path path = new Path(destPath, String.format("part-r-%05d", part++)); - FileSystem ns = fs.getFileSystemByPath(path); + FileSystem ns = fs.getVolumeByPath(path).getFileSystem(); @SuppressWarnings("deprecation") MapFile.Writer output = new MapFile.Writer(ns.getConf(), ns, path.toString(), LogFileKey.class, LogFileValue.class); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java index a28bac4..541f075 100644 --- a/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java +++ b/server/tserver/src/main/java/org/apache/accumulo/tserver/log/MultiReader.java @@ -97,7 +97,7 @@ public class MultiReader { foundFinish = true; continue; } - FileSystem ns = fs.getFileSystemByPath(child.getPath()); + FileSystem ns = fs.getVolumeByPath(child.getPath()).getFileSystem(); heap.add(new Index(new Reader(ns, child.getPath().toString(), ns.getConf()))); } if (!foundFinish) http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java index 1cd8f12..7a1c84b 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/RootFilesTest.java @@ -116,6 +116,7 @@ public class RootFilesTest { ConfigurationCopy conf = new ConfigurationCopy(); conf.set(Property.INSTANCE_DFS_URI, "file:///"); + conf.set(Property.INSTANCE_DFS_DIR, "/"); VolumeManager vm = VolumeManagerImpl.get(conf); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java index 50c8b31..dad9a75 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/TabletServerSyncCheckTest.java @@ -22,6 +22,8 @@ import java.util.Map; import org.apache.accumulo.core.conf.ConfigurationCopy; import org.apache.accumulo.core.data.Key; +import org.apache.accumulo.core.volume.Volume; +import org.apache.accumulo.core.volume.VolumeImpl; import org.apache.accumulo.server.fs.VolumeManagerImpl; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.ContentSummary; @@ -44,7 +46,7 @@ public class TabletServerSyncCheckTest { conf.set(DFS_DURABLE_SYNC, "false"); FileSystem fs = new TestFileSystem(conf); - TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs)); + TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/"))); vm.ensureSyncIsEnabled(); } @@ -56,7 +58,7 @@ public class TabletServerSyncCheckTest { FileSystem fs1 = new TestFileSystem(conf1); FileSystem fs2 = new TestFileSystem(conf2); - TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("bar", fs2, "foo", fs1)); + TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("bar", new VolumeImpl(fs2, "/"), "foo", new VolumeImpl(fs1, "/"))); vm.ensureSyncIsEnabled(); } @@ -67,7 +69,7 @@ public class TabletServerSyncCheckTest { conf.set(DFS_SUPPORT_APPEND, "false"); FileSystem fs = new TestFileSystem(conf); - TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.of("foo", fs)); + TestVolumeManagerImpl vm = new TestVolumeManagerImpl(ImmutableMap.<String,Volume> of("foo", new VolumeImpl(fs, "/"))); vm.ensureSyncIsEnabled(); } @@ -89,8 +91,8 @@ public class TabletServerSyncCheckTest { private class TestVolumeManagerImpl extends VolumeManagerImpl { - public TestVolumeManagerImpl(Map<String,? extends FileSystem> volumes) { - super(volumes, volumes.keySet().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap())); + public TestVolumeManagerImpl(Map<String,Volume> volumes) { + super(volumes, volumes.values().iterator().next(), new ConfigurationCopy(Collections.<String,String> emptyMap())); } @Override @@ -149,7 +151,7 @@ public class TabletServerSyncCheckTest { } @Override - public FileSystem getFileSystemByPath(Path path) { + public Volume getVolumeByPath(Path path) { return null; } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java index c4d3dfb..a79e77e 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/MultiReaderTest.java @@ -42,13 +42,13 @@ public class MultiReaderTest { @Before public void setUp() throws Exception { - fs = VolumeManagerImpl.getLocal(); root.create(); - String path = root.getRoot().getAbsolutePath(); - Path root = new Path("file://" + path + "/manyMaps"); + String path = root.getRoot().getAbsolutePath() + "/manyMaps"; + fs = VolumeManagerImpl.getLocal(path); + Path root = new Path("file://" + path); fs.mkdirs(root); fs.create(new Path(root, "finished")).close(); - FileSystem ns = fs.getFileSystemByPath(root); + FileSystem ns = fs.getVolumeByPath(root).getFileSystem(); @SuppressWarnings("deprecation") Writer oddWriter = new Writer(ns.getConf(), ns, new Path(root, "odd").toString(), IntWritable.class, BytesWritable.class); http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java index 359bfa1..fffa15e 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/SortedLogRecoveryTest.java @@ -117,14 +117,15 @@ public class SortedLogRecoveryTest { private static List<Mutation> recover(Map<String,KeyValue[]> logs, Set<String> files, KeyExtent extent) throws IOException { TemporaryFolder root = new TemporaryFolder(new File(System.getProperty("user.dir") + "/target")); root.create(); - final String workdir = "file://" + root.getRoot().getAbsolutePath() + "/workdir"; - VolumeManager fs = VolumeManagerImpl.getLocal(); - fs.deleteRecursively(new Path(workdir)); + final String workdir = root.getRoot().getAbsolutePath() + "/workdir"; + VolumeManager fs = VolumeManagerImpl.getLocal(workdir); + final Path workdirPath = new Path("file://" + workdir); + fs.deleteRecursively(workdirPath); ArrayList<Path> dirs = new ArrayList<Path>(); try { for (Entry<String,KeyValue[]> entry : logs.entrySet()) { String path = workdir + "/" + entry.getKey(); - FileSystem ns = fs.getFileSystemByPath(new Path(path)); + FileSystem ns = fs.getVolumeByPath(new Path(path)).getFileSystem(); @SuppressWarnings("deprecation") Writer map = new MapFile.Writer(ns.getConf(), ns, path + "/log1", LogFileKey.class, LogFileValue.class); for (KeyValue lfe : entry.getValue()) { http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java index af149fa..d6c23e3 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/log/TestUpgradePathForWALogs.java @@ -61,10 +61,10 @@ public class TestUpgradePathForWALogs { public void setUp() throws Exception { // quiet log messages about compress.CodecPool Logger.getRootLogger().setLevel(Level.ERROR); - fs = VolumeManagerImpl.getLocal(); root.create(); - String path = root.getRoot().getAbsolutePath(); - Path manyMapsPath = new Path("file://" + path + "/manyMaps"); + String path = root.getRoot().getAbsolutePath() + "/manyMaps"; + fs = VolumeManagerImpl.getLocal(path); + Path manyMapsPath = new Path("file://" + path); fs.mkdirs(manyMapsPath); fs.create(new Path(manyMapsPath, "finished")).close(); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java index 9a9cad7..d2c8d0f 100644 --- a/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java +++ b/test/src/main/java/org/apache/accumulo/test/performance/scan/CollectTabletStats.java @@ -398,7 +398,7 @@ public class CollectTabletStats { // assume it is a map file status = fs.getFileStatus(new Path(file + "/data")); } - FileSystem ns = fs.getFileSystemByPath(file.path()); + FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem(); BlockLocation[] locs = ns.getFileBlockLocations(status, 0, status.getLen()); System.out.println("\t\t\tBlocks for : " + file); @@ -445,7 +445,7 @@ public class CollectTabletStats { HashSet<ByteSequence> columnSet = createColumnBSS(columns); for (FileRef file : files) { - FileSystem ns = fs.getFileSystemByPath(file.path()); + FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem(); FileSKVIterator reader = FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf); Range range = new Range(ke.getPrevEndRow(), false, ke.getEndRow(), true); reader.seek(range, columnSet, columnSet.size() == 0 ? false : true); @@ -475,7 +475,7 @@ public class CollectTabletStats { List<SortedKeyValueIterator<Key,Value>> readers = new ArrayList<SortedKeyValueIterator<Key,Value>>(files.size()); for (FileRef file : files) { - FileSystem ns = fs.getFileSystemByPath(file.path()); + FileSystem ns = fs.getVolumeByPath(file.path()).getFileSystem(); readers.add(FileOperations.getInstance().openReader(file.path().toString(), false, ns, ns.getConf(), aconf.getConfiguration())); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/test/java/org/apache/accumulo/test/VolumeIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java index a0efe45..a7f7556 100644 --- a/test/src/test/java/org/apache/accumulo/test/VolumeIT.java +++ b/test/src/test/java/org/apache/accumulo/test/VolumeIT.java @@ -23,6 +23,7 @@ import java.io.BufferedOutputStream; import java.io.File; import java.io.FileOutputStream; import java.io.IOException; +import java.net.URI; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -104,8 +105,9 @@ public class VolumeIT extends ConfigurableMacIT { @Override public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { // Run MAC on two locations in the local file system - cfg.setProperty(Property.INSTANCE_DFS_URI, v1.toString()); - cfg.setProperty(Property.INSTANCE_DFS_DIR, "/accumulo"); + URI v1Uri = v1.toUri(); + cfg.setProperty(Property.INSTANCE_DFS_DIR, v1Uri.getPath()); + cfg.setProperty(Property.INSTANCE_DFS_URI, v1Uri.getScheme() + v1Uri.getHost()); cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString() + "," + v2.toString()); // use raw local file system so walogs sync and flush will work @@ -114,7 +116,7 @@ public class VolumeIT extends ConfigurableMacIT { super.configure(cfg, hadoopCoreSite); } - @Test + @Test(timeout = 2 * 60 * 1000) public void test() throws Exception { // create a table Connector connector = getConnector(); @@ -176,7 +178,7 @@ public class VolumeIT extends ConfigurableMacIT { Assert.assertEquals(expected, actual); } - @Test + @Test(timeout = 2 * 60 * 1000) public void testRelativePaths() throws Exception { List<String> expected = new ArrayList<String>(); @@ -292,9 +294,8 @@ public class VolumeIT extends ConfigurableMacIT { // check that all volumes are initialized for (Path volumePath : Arrays.asList(v1, v2, v3)) { FileSystem fs = volumePath.getFileSystem(CachedConfiguration.getInstance()); - Path vp = new Path(volumePath, "accumulo"); - Path vpi = new Path(vp, ServerConstants.INSTANCE_ID_DIR); - FileStatus[] iids = fs.listStatus(vpi); + Path vp = new Path(volumePath, ServerConstants.INSTANCE_ID_DIR); + FileStatus[] iids = fs.listStatus(vp); Assert.assertEquals(1, iids.length); Assert.assertEquals(uuid, iids[0].getPath().getName()); } @@ -390,7 +391,7 @@ public class VolumeIT extends ConfigurableMacIT { Assert.assertEquals(200, sum); } - @Test + @Test(timeout = 5 * 60 * 1000) public void testRemoveVolumes() throws Exception { String[] tableNames = getTableNames(2); @@ -447,12 +448,12 @@ public class VolumeIT extends ConfigurableMacIT { File v1f = new File(v1.toUri()); File v8f = new File(new File(v1.getParent().toUri()), "v8"); - v1f.renameTo(v8f); + Assert.assertTrue("Failed to rename " + v1f + " to " + v8f, v1f.renameTo(v8f)); Path v8 = new Path(v8f.toURI()); File v2f = new File(v2.toUri()); File v9f = new File(new File(v2.getParent().toUri()), "v9"); - v2f.renameTo(v9f); + Assert.assertTrue("Failed to rename " + v2f + " to " + v9f, v2f.renameTo(v9f)); Path v9 = new Path(v9f.toURI()); Configuration conf = new Configuration(false); @@ -494,12 +495,12 @@ public class VolumeIT extends ConfigurableMacIT { verifyVolumesUsed(tableNames[2], true, v8, v9); } - @Test + @Test(timeout = 5 * 60 * 1000) public void testCleanReplaceVolumes() throws Exception { testReplaceVolume(true); } - @Test + @Test(timeout = 5 * 60 * 1000) public void testDirtyReplaceVolumes() throws Exception { testReplaceVolume(false); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/7c94c086/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java index c8023c0..eee093b 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/BulkFileIT.java @@ -29,9 +29,9 @@ import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.Value; import org.apache.accumulo.core.file.FileOperations; import org.apache.accumulo.core.file.FileSKVWriter; -import org.apache.accumulo.core.file.VolumeConfiguration; import org.apache.accumulo.core.file.rfile.RFile; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.core.volume.VolumeConfiguration; import org.apache.accumulo.server.conf.ServerConfiguration; import org.apache.accumulo.server.trace.TraceFileSystem; import org.apache.hadoop.conf.Configuration; @@ -53,7 +53,7 @@ public class BulkFileIT extends SimpleMacIT { c.tableOperations().addSplits(tableName, splits); Configuration conf = new Configuration(); AccumuloConfiguration aconf = ServerConfiguration.getDefaultConfiguration(); - FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultFilesystem(conf, aconf)); + FileSystem fs = TraceFileSystem.wrap(VolumeConfiguration.getDefaultVolume(conf, aconf).getFileSystem()); String dir = rootPath() + "/bulk_test_diff_files_89723987592_" + getTableNames(1)[0];
