ACCUMULO-2061 Remove implicit assumptions about '/accumulo' directory that all volumes would have.
Added some extra logging at trace to help if this needs to be revisited. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/039989ad Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/039989ad Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/039989ad Branch: refs/heads/ACCUMULO-2061 Commit: 039989ad44bb9b46d620a3e302e15c32b3b3840a Parents: c3214e0 Author: Josh Elser <els...@apache.org> Authored: Tue Mar 11 13:57:48 2014 -0400 Committer: Josh Elser <els...@apache.org> Committed: Tue Mar 11 13:57:48 2014 -0400 ---------------------------------------------------------------------- .../apache/accumulo/server/ServerConstants.java | 4 +- .../accumulo/server/fs/VolumeManager.java | 3 +- .../apache/accumulo/server/fs/VolumeUtil.java | 20 ++++- .../accumulo/server/fs/VolumeUtilTest.java | 89 ++++++++++++++++++-- 4 files changed, 103 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java index 6c27897..7dd0a08 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java +++ b/server/base/src/main/java/org/apache/accumulo/server/ServerConstants.java @@ -187,8 +187,8 @@ public class ServerConstants { HashSet<Path> baseDirs = new HashSet<Path>(); for (String baseDir : getBaseUris()) { - // normalize using path and remove accumulo dir - baseDirs.add(new Path(baseDir).getParent()); + // normalize using path + baseDirs.add(new Path(baseDir)); } for (Pair<Path,Path> pair : ret) http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java index c345ede..9b8fb98 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManager.java @@ -48,9 +48,10 @@ public interface VolumeManager { } private static int endOfVolumeIndex(String path, String dir) { + // Strip off the suffix that starts with the FileType (e.g. tables, wal, etc) int dirIndex = path.indexOf('/' + dir); if (dirIndex != -1) { - return path.lastIndexOf('/', dirIndex - 1); + return dirIndex; } if (path.contains(":")) http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java index 110f1e4..bcfb008 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/fs/VolumeUtil.java @@ -81,8 +81,10 @@ public class VolumeUtil { } public static String switchVolume(String path, FileType ft, List<Pair<Path,Path>> replacements) { - if (replacements.size() == 0) + if (replacements.size() == 0) { + log.trace("Not switching volume because there are no replacements"); return null; + } Path p = new Path(path); @@ -92,10 +94,15 @@ public class VolumeUtil { for (Pair<Path,Path> pair : replacements) { Path key = removeTrailingSlash(pair.getFirst()); - if (key.equals(volume)) - return new Path(pair.getSecond(), ft.removeVolume(p)).toString(); + if (key.equals(volume)) { + String replacement = new Path(pair.getSecond(), ft.removeVolume(p)).toString(); + log.trace("Replacing " + path + " with " + replacement); + return replacement; + } } + log.trace("Could not find replacement for " + ft + " at " + path); + return null; } @@ -119,13 +126,17 @@ public class VolumeUtil { } - if (numSwitched == 0) + if (numSwitched == 0) { + log.trace("Did not switch " + le); return null; + } LogEntry newLogEntry = new LogEntry(le); newLogEntry.filename = switchedPath; newLogEntry.logSet = switchedLogs; + log.trace("Switched " + le + " to " + newLogEntry); + return newLogEntry; } @@ -165,6 +176,7 @@ public class VolumeUtil { */ public static TabletFiles updateTabletVolumes(ZooLock zooLock, VolumeManager vm, KeyExtent extent, TabletFiles tabletFiles) throws IOException { List<Pair<Path,Path>> replacements = ServerConstants.getVolumeReplacements(); + log.trace("Using volume replacements: " + replacements); List<LogEntry> logsToRemove = new ArrayList<LogEntry>(); List<LogEntry> logsToAdd = new ArrayList<LogEntry>(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/039989ad/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java ---------------------------------------------------------------------- diff --git a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java index c85be45..3b905c9 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/fs/VolumeUtilTest.java @@ -43,9 +43,9 @@ public class VolumeUtilTest { @Test public void testSwitchVolume() { List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/"), new Path("viewfs:/b"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b/accumulo"))); Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf", VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); @@ -57,9 +57,9 @@ public class VolumeUtilTest { Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); replacements.clear(); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1"), new Path("viewfs:/a"))); - replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/"), new Path("viewfs:/b"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a/accumulo"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b/accumulo"))); Assert.assertEquals("viewfs:/a/accumulo/tables/t-00000/C000.rf", VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); @@ -73,6 +73,70 @@ public class VolumeUtilTest { } @Test + public void testSwitchVolumesDifferentSourceDepths() { + List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/b"))); + + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + + replacements.clear(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/a"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/b"))); + + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/a/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/b/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/a/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + } + + @Test + public void testSwitchVolumesDifferentTargetDepths() { + List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/accumulo"), new Path("viewfs:/path3"))); + + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + + replacements.clear(); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1/d1/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn1:9000/d1/accumulo"), new Path("viewfs:/path1/path2"))); + replacements.add(new Pair<Path,Path>(new Path("hdfs://nn2/d2/accumulo"), new Path("viewfs:/path3"))); + + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path1/path2/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn1:9000/d1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertEquals("viewfs:/path3/tables/t-00000/C000.rf", + VolumeUtil.switchVolume("hdfs://nn2/d2/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("viewfs:/path1/path2/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("file:/nn1/a/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + Assert.assertNull(VolumeUtil.switchVolume("hdfs://nn1/accumulo/tables/t-00000/C000.rf", FileType.TABLE, replacements)); + } + + @Test public void testSame() throws Exception { FileSystem fs = FileSystem.getLocal(new Configuration()); @@ -133,6 +197,19 @@ public class VolumeUtilTest { } + @Test + public void testRootTableReplacement() throws IOException { + List<Pair<Path,Path>> replacements = new ArrayList<Pair<Path,Path>>(); + replacements.add(new Pair<Path,Path>(new Path("file:/foo/v1"), new Path("file:/foo/v8"))); + replacements.add(new Pair<Path,Path>(new Path("file:/foo/v2"), new Path("file:/foo/v9"))); + + FileType ft = FileType.TABLE; + + Assert.assertEquals("file:/foo/v8/tables/+r/root_tablet", + VolumeUtil.switchVolume("file:/foo/v1/tables/+r/root_tablet", + ft, replacements)); + } + private void writeFile(FileSystem fs, Path dir, String filename, String data) throws IOException { FSDataOutputStream out = fs.create(new Path(dir, filename)); try {
