Repository: accumulo Updated Branches: refs/heads/master 0dda8a511 -> edf659544
ACCUMULO-2455 small work arounds to increase the chance that zookeeper updates are in place before tservers need to see them Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/09307322 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/09307322 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/09307322 Branch: refs/heads/master Commit: 09307322a3e9f45cd37bbd4c63fa66ccbe2cc0a2 Parents: 46bf772 Author: Eric Newton <eric.new...@gmail.com> Authored: Tue Mar 11 16:30:45 2014 -0400 Committer: Eric Newton <eric.new...@gmail.com> Committed: Tue Mar 11 16:30:45 2014 -0400 ---------------------------------------------------------------------- .../apache/accumulo/proxy/SimpleProxyIT.java | 13 ++++----- .../org/apache/accumulo/test/NamespacesIT.java | 8 +++--- .../accumulo/test/functional/BloomFilterIT.java | 30 ++++++++++++-------- 3 files changed, 28 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java ---------------------------------------------------------------------- diff --git a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java index efe36b9..f554d63 100644 --- a/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java +++ b/proxy/src/test/java/org/apache/accumulo/proxy/SimpleProxyIT.java @@ -772,15 +772,14 @@ public class SimpleProxyIT { // create a table that's very slow, so we can look for scans/compactions client.createTable(creds, "slow", true, TimeType.MILLIS); + IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250")); + client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class)); - // Should take 5 seconds to read every record - for (int i = 0; i < 20; i++) { + // Should take 10 seconds to read every record + for (int i = 0; i < 40; i++) { client.updateAndFlush(creds, "slow", mutation("row" + i, "cf", "cq", "value")); } - IteratorSetting setting = new IteratorSetting(100, "slow", SlowIterator.class.getName(), Collections.singletonMap("sleepTime", "250")); - client.attachIterator(creds, "slow", setting, EnumSet.allOf(IteratorScope.class)); - // scan Thread t = new Thread() { @Override @@ -798,7 +797,7 @@ public class SimpleProxyIT { }; t.start(); - // look for the scan + // look for the scan many times List<ActiveScan> scans = new ArrayList<ActiveScan>(); for (int i = 0; i < 100 && scans.isEmpty(); i++) { for (String tserver : client.getTabletServers(creds)) { @@ -811,7 +810,7 @@ public class SimpleProxyIT { if (!scans.isEmpty()) break; - UtilWaitThread.sleep(10); + UtilWaitThread.sleep(100); } } t.join(); http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java index a8d5f7f..af9274b 100644 --- a/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java +++ b/test/src/test/java/org/apache/accumulo/test/NamespacesIT.java @@ -321,17 +321,17 @@ public class NamespacesIT extends SimpleMacIT { } IteratorSetting setting2 = c.namespaceOperations().getIteratorSetting(namespace, setting.getName(), IteratorScope.scan); assertEquals(setting, setting2); - s = c.createScanner(t1, Authorizations.EMPTY); - assertFalse(s.iterator().hasNext()); assertTrue(c.namespaceOperations().listIterators(namespace).containsKey(iterName)); assertTrue(c.tableOperations().listIterators(t1).containsKey(iterName)); + s = c.createScanner(t1, Authorizations.EMPTY); + assertFalse(s.iterator().hasNext()); // verify can see inserted entry again c.namespaceOperations().removeIterator(namespace, setting.getName(), EnumSet.allOf(IteratorScope.class)); - s = c.createScanner(t1, Authorizations.EMPTY); - assertTrue(s.iterator().hasNext()); assertFalse(c.namespaceOperations().listIterators(namespace).containsKey(iterName)); assertFalse(c.tableOperations().listIterators(t1).containsKey(iterName)); + s = c.createScanner(t1, Authorizations.EMPTY); + assertTrue(s.iterator().hasNext()); } @Test http://git-wip-us.apache.org/repos/asf/accumulo/blob/09307322/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java index 27aab91..50ca776 100644 --- a/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java +++ b/test/src/test/java/org/apache/accumulo/test/functional/BloomFilterIT.java @@ -39,6 +39,7 @@ import org.apache.accumulo.core.file.keyfunctor.ColumnFamilyFunctor; import org.apache.accumulo.core.file.keyfunctor.ColumnQualifierFunctor; import org.apache.accumulo.core.file.keyfunctor.RowFunctor; import org.apache.accumulo.core.security.Authorizations; +import org.apache.accumulo.fate.util.UtilWaitThread; import org.apache.accumulo.minicluster.MemoryUnit; import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; import org.apache.hadoop.conf.Configuration; @@ -56,6 +57,7 @@ public class BloomFilterIT extends ConfigurableMacIT { siteConfig.put(Property.TABLE_BLOOM_SIZE.getKey(), "2000000"); siteConfig.put(Property.TABLE_BLOOM_ERRORRATE.getKey(), "1%"); siteConfig.put(Property.TABLE_BLOOM_LOAD_THRESHOLD.getKey(), "0"); + siteConfig.put(Property.TSERV_MUTATION_QUEUE_MAX.getKey(), "10M"); siteConfig.put(Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1G"); cfg.setSiteConfig(siteConfig ); } @@ -69,9 +71,9 @@ public class BloomFilterIT extends ConfigurableMacIT { c.tableOperations().setProperty(table, Property.TABLE_BLOCKCACHE_ENABLED.getKey(), "false"); } log.info("Writing"); - write(c, "bt1", 1, 0, 2000000000, 1000); - write(c, "bt2", 2, 0, 2000000000, 1000); - write(c, "bt3", 3, 0, 2000000000, 1000); + write(c, "bt1", 1, 0, 2000000000, 500); + write(c, "bt2", 2, 0, 2000000000, 500); + write(c, "bt3", 3, 0, 2000000000, 500); log.info("Writing complete"); // test inserting an empty key @@ -94,35 +96,39 @@ public class BloomFilterIT extends ConfigurableMacIT { // these queries should only run quickly if bloom filters are working, so lets get a base log.info("Base query"); - long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000); - long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000); - long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000); + long t1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500); + long t2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500); + long t3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500); log.info("Base query complete"); log.info("Rewriting with bloom filters"); c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_ENABLED.getKey(), "true"); c.tableOperations().setProperty("bt1", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName()); - c.tableOperations().compact("bt1", null, null, false, true); c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_ENABLED.getKey(), "true"); c.tableOperations().setProperty("bt2", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnFamilyFunctor.class.getName()); - c.tableOperations().compact("bt2", null, null, false, true); c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_ENABLED.getKey(), "true"); c.tableOperations().setProperty("bt3", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), ColumnQualifierFunctor.class.getName()); - c.tableOperations().compact("bt3", null, null, false, true); c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_ENABLED.getKey(), "true"); c.tableOperations().setProperty("bt4", Property.TABLE_BLOOM_KEY_FUNCTOR.getKey(), RowFunctor.class.getName()); + + // ensure the updates to zookeeper propogate + UtilWaitThread.sleep(500); + c.tableOperations().compact("bt4", null, null, false, true); + c.tableOperations().compact("bt1", null, null, false, true); + c.tableOperations().compact("bt2", null, null, false, true); + c.tableOperations().compact("bt3", null, null, false, true); log.info("Rewriting with bloom filters complete"); // these queries should only run quickly if bloom // filters are working log.info("Bloom query"); - long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 1000); - long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 1000); - long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 1000); + long tb1 = query(c, "bt1", 1, 0, 2000000000, 100000, 500); + long tb2 = query(c, "bt2", 2, 0, 2000000000, 100000, 500); + long tb3 = query(c, "bt3", 3, 0, 2000000000, 100000, 500); log.info("Bloom query complete"); timeCheck(t1 + t2 + t3, tb1 + tb2 + tb3);
