Repository: accumulo Updated Branches: refs/heads/1.6 fa978b0c3 -> 4b87abd48 refs/heads/master b5d7b9485 -> 5fdbb8191
ACCUMULO-3314 Add a fair volume chooser to stabilize RewriteTabletDirectoriesIT The RandomVolumeChooser creates failure potential for the test due to its random nature. A volume chooser that is stable should result in a stable test. Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/62e3e9b5 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/62e3e9b5 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/62e3e9b5 Branch: refs/heads/1.6 Commit: 62e3e9b53b35da98c9fff32a1366911c94dd8c7b Parents: fa978b0 Author: Josh Elser <els...@apache.org> Authored: Fri Nov 7 13:13:30 2014 -0500 Committer: Josh Elser <els...@apache.org> Committed: Fri Nov 7 13:13:30 2014 -0500 ---------------------------------------------------------------------- .../apache/accumulo/test/FairVolumeChooser.java | 48 ++++++++++++++++++++ .../test/RewriteTabletDirectoriesIT.java | 19 ++++++-- 2 files changed, 63 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/62e3e9b5/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java ---------------------------------------------------------------------- diff --git a/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java b/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java new file mode 100644 index 0000000..9eb0c84 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/FairVolumeChooser.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.test; + +import java.util.concurrent.ConcurrentHashMap; + +import org.apache.accumulo.server.fs.VolumeChooser; + +/** + * Try to assign some fairness to choosing Volumes. Intended for tests, not for production + */ +public class FairVolumeChooser implements VolumeChooser { + + private final ConcurrentHashMap<Integer,Integer> optionLengthToLastChoice = new ConcurrentHashMap<Integer,Integer>(); + + @Override + public String choose(String[] options) { + int currentChoice; + Integer lastChoice = optionLengthToLastChoice.get(options.length); + if (null == lastChoice) { + currentChoice = 0; + } else { + currentChoice = lastChoice + 1; + if (currentChoice >= options.length) { + currentChoice = 0; + } + } + + optionLengthToLastChoice.put(options.length, currentChoice); + + return options[currentChoice]; + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/62e3e9b5/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java index fe233ee..cacee7a 100644 --- a/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java +++ b/test/src/test/java/org/apache/accumulo/test/RewriteTabletDirectoriesIT.java @@ -18,6 +18,7 @@ package org.apache.accumulo.test; import static org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection.ServerColumnFamily.DIRECTORY_COLUMN; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import java.io.BufferedOutputStream; @@ -52,7 +53,7 @@ import org.junit.Test; // ACCUMULO-3263 public class RewriteTabletDirectoriesIT extends ConfigurableMacIT { - + @Override public int defaultTimeoutSeconds() { return 4 * 60; @@ -71,6 +72,8 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT { v1 = new Path("file://" + v1f.getAbsolutePath()); v2 = new Path("file://" + v2f.getAbsolutePath()); + // Use a VolumeChooser which should be more fair + cfg.setProperty(Property.GENERAL_VOLUME_CHOOSER, FairVolumeChooser.class.getName()); // Run MAC on two locations in the local file system cfg.setProperty(Property.INSTANCE_VOLUMES, v1.toString()); hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); @@ -83,6 +86,8 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT { c.securityOperations().grantTablePermission(c.whoami(), MetadataTable.NAME, TablePermission.WRITE); final String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); + + // Write some data to a table and add some splits BatchWriter bw = c.createBatchWriter(tableName, null); final SortedSet<Text> splits = new TreeSet<Text>(); for (String split : "a,b,c,d,e,f,g,h,i,j,k,l,m,n,o,p,q,r,s,t,u,v,w,x,y,z".split(",")) { @@ -97,12 +102,13 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT { BatchScanner scanner = c.createBatchScanner(MetadataTable.NAME, Authorizations.EMPTY, 1); DIRECTORY_COLUMN.fetch(scanner); String tableId = c.tableOperations().tableIdMap().get(tableName); + assertNotNull("TableID for " + tableName + " was null", tableId); scanner.setRanges(Collections.singletonList(TabletsSection.getRange(tableId))); // verify the directory entries are all on v1, make a few entries relative bw = c.createBatchWriter(MetadataTable.NAME, null); int count = 0; for (Entry<Key,Value> entry : scanner) { - assertTrue(entry.getValue().toString().contains(v1.toString())); + assertTrue("Expected " + entry.getValue() + " to contain " + v1, entry.getValue().toString().contains(v1.toString())); count++; if (count % 2 == 0) { String parts[] = entry.getValue().toString().split("/"); @@ -146,12 +152,17 @@ public class RewriteTabletDirectoriesIT extends ConfigurableMacIT { v2Count++; } } + + log.info("Count for volume1: " + v1Count); + log.info("Count for volume2: " + v2Count); + assertEquals(splits.size() + 1, v1Count + v2Count); - assertTrue(Math.abs(v1Count - v2Count) < 10); + // a fair chooser will differ by less than count(volumes) + assertTrue("Expected the number of files to differ between volumes by less than 10. " + v1Count + " " + v2Count, Math.abs(v1Count - v2Count) < 2); // verify we can read the old data count = 0; for (Entry<Key,Value> entry : c.createScanner(tableName, Authorizations.EMPTY)) { - assertTrue(splits.contains(entry.getKey().getRow())); + assertTrue("Found unexpected entry in table: " + entry, splits.contains(entry.getKey().getRow())); count++; } assertEquals(splits.size(), count);
