ACCUMULO-3777 Add existence check to MemoryManager Somehow we got into a situation where we were checking tablets for a deleted table. Add a quick check for existence before running the extent through the memory manager.
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/719aac43 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/719aac43 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/719aac43 Branch: refs/heads/master Commit: 719aac433e0428f969130820427d62404d5642e2 Parents: 757c1cb Author: Josh Elser <els...@apache.org> Authored: Sun May 10 01:09:19 2015 -0400 Committer: Josh Elser <els...@apache.org> Committed: Sun May 10 01:09:19 2015 -0400 ---------------------------------------------------------------------- .../tabletserver/LargestFirstMemoryManager.java | 13 ++++ .../LargestFirstMemoryManagerTest.java | 67 ++++++++++++++++++++ 2 files changed, 80 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java index 323e59d..a39c8b6 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java @@ -22,6 +22,8 @@ import java.util.List; import java.util.Map.Entry; import java.util.TreeMap; +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.impl.Tables; import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.impl.KeyExtent; import org.apache.accumulo.server.conf.ServerConfiguration; @@ -143,11 +145,16 @@ public class LargestFirstMemoryManager implements MemoryManager { return mincIdleThresholds.get(tableId); } + boolean tableExists(Instance instance, String tableId) { + return Tables.exists(instance, tableId); + } + @Override public MemoryManagementActions getMemoryManagementActions(List<TabletState> tablets) { if (maxMemory < 0) throw new IllegalStateException("need to initialize " + LargestFirstMemoryManager.class.getName()); + final Instance instance = config.getInstance(); final int maxMinCs = maxConcurrentMincs * numWaitingMultiplier; mincIdleThresholds.clear(); @@ -164,6 +171,12 @@ public class LargestFirstMemoryManager implements MemoryManager { // find the largest and most idle tablets for (TabletState ts : tablets) { + // Make sure that the table still exists + if (!tableExists(instance, ts.getExtent().getTableId().toString())) { + log.info("Ignoring extent for deleted table: " + ts.getExtent()); + continue; + } + final long memTabletSize = ts.getMemTableSize(); final long minorCompactingSize = ts.getMinorCompactingMemTableSize(); final long idleTime = now - Math.max(ts.getLastCommitTime(), ZERO_TIME); http://git-wip-us.apache.org/repos/asf/accumulo/blob/719aac43/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java index 7c15cfc..b08b980 100644 --- a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java +++ b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java @@ -33,6 +33,8 @@ import org.apache.accumulo.server.conf.TableConfiguration; import org.apache.hadoop.io.Text; import org.junit.Test; +import com.google.common.base.Function; + public class LargestFirstMemoryManagerTest { private static final long ZERO = System.currentTimeMillis(); @@ -160,6 +162,52 @@ public class LargestFirstMemoryManagerTest { assertEquals(k("b"), result.tabletsToMinorCompact.get(0)); } + @Test + public void testDeletedTable() throws Exception { + final String deletedTableId = "1"; + Function<String,Boolean> existenceCheck = new Function<String,Boolean>() { + public Boolean apply(String tableId) { + return !deletedTableId.equals(tableId); + } + }; + LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck); + ServerConfiguration config = new ServerConfiguration() { + ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance()); + + @Override + public AccumuloConfiguration getConfiguration() { + return DefaultConfiguration.getInstance(); + } + + @Override + public TableConfiguration getTableConfiguration(String tableId) { + return delegate.getTableConfiguration(tableId); + } + + @Override + public TableConfiguration getTableConfiguration(KeyExtent extent) { + return delegate.getTableConfiguration(extent); + } + + @Override + public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) { + return delegate.getNamespaceConfiguration(namespaceId); + } + + @Override + public Instance getInstance() { + return delegate.getInstance(); + } + }; + mgr.init(config); + MemoryManagementActions result; + // one tablet is really big and the other is for a nonexistent table + KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null); + result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(extent, result.tabletsToMinorCompact.get(0)); + } + private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager { public long currentTime = ZERO; @@ -174,6 +222,25 @@ public class LargestFirstMemoryManagerTest { return 15 * 60 * 1000; } + @Override + boolean tableExists(Instance instance, String tableId) { + return true; + } + } + + private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest { + + Function<String,Boolean> existenceCheck; + + public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck) { + super(); + this.existenceCheck = existenceCheck; + } + + @Override + boolean tableExists(Instance instance, String tableId) { + return existenceCheck.apply(tableId); + } } private static KeyExtent k(String endRow) {
