ACCUMULO-4312 Fix additional sealing problems in 1.7 branch
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d10840b8 Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d10840b8 Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d10840b8 Branch: refs/heads/1.7 Commit: d10840b8e5177f1f026f28705f93943a1921ad82 Parents: 544d661 Author: Christopher Tubbs <ctubb...@apache.org> Authored: Fri May 27 15:59:35 2016 -0400 Committer: Christopher Tubbs <ctubb...@apache.org> Committed: Fri May 27 15:59:35 2016 -0400 ---------------------------------------------------------------------- .../tabletserver/LargestFirstMemoryManager.java | 2 +- .../LargestFirstMemoryManagerTest.java | 294 --- .../tserver/LargestFirstMemoryManagerTest.java | 298 +++ .../accumulo/proxy/ProxyDurabilityIT.java | 139 -- .../apache/accumulo/proxy/SimpleProxyBase.java | 2272 ----------------- .../apache/accumulo/proxy/TBinaryProxyIT.java | 33 - .../apache/accumulo/proxy/TCompactProxyIT.java | 32 - .../accumulo/proxy/TJsonProtocolProxyIT.java | 33 - .../apache/accumulo/proxy/TTupleProxyIT.java | 33 - .../proxy/TestProxyInstanceOperations.java | 83 - .../accumulo/proxy/TestProxyReadWrite.java | 466 ---- .../proxy/TestProxySecurityOperations.java | 146 -- .../proxy/TestProxyTableOperations.java | 201 -- .../accumulo/test/proxy/ProxyDurabilityIT.java | 141 ++ .../accumulo/test/proxy/SimpleProxyBase.java | 2274 ++++++++++++++++++ .../accumulo/test/proxy/TBinaryProxyIT.java | 33 + .../accumulo/test/proxy/TCompactProxyIT.java | 32 + .../test/proxy/TJsonProtocolProxyIT.java | 33 + .../accumulo/test/proxy/TTupleProxyIT.java | 33 + .../test/proxy/TestProxyInstanceOperations.java | 85 + .../accumulo/test/proxy/TestProxyReadWrite.java | 469 ++++ .../test/proxy/TestProxySecurityOperations.java | 148 ++ .../test/proxy/TestProxyTableOperations.java | 203 ++ 23 files changed, 3750 insertions(+), 3733 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java ---------------------------------------------------------------------- diff --git a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java index d73c46e..e49e1af 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java +++ b/server/base/src/main/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManager.java @@ -146,7 +146,7 @@ public class LargestFirstMemoryManager implements MemoryManager { return mincIdleThresholds.get(tableId); } - boolean tableExists(Instance instance, String tableId) { + protected boolean tableExists(Instance instance, String tableId) { return Tables.exists(instance, tableId); } http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java deleted file mode 100644 index b08b980..0000000 --- a/server/tserver/src/test/java/org/apache/accumulo/server/tabletserver/LargestFirstMemoryManagerTest.java +++ /dev/null @@ -1,294 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.server.tabletserver; - -import static org.junit.Assert.assertEquals; - -import java.util.Arrays; -import java.util.List; - -import org.apache.accumulo.core.client.Instance; -import org.apache.accumulo.core.client.mock.MockInstance; -import org.apache.accumulo.core.conf.AccumuloConfiguration; -import org.apache.accumulo.core.conf.DefaultConfiguration; -import org.apache.accumulo.core.data.impl.KeyExtent; -import org.apache.accumulo.server.conf.NamespaceConfiguration; -import org.apache.accumulo.server.conf.ServerConfiguration; -import org.apache.accumulo.server.conf.ServerConfigurationFactory; -import org.apache.accumulo.server.conf.TableConfiguration; -import org.apache.hadoop.io.Text; -import org.junit.Test; - -import com.google.common.base.Function; - -public class LargestFirstMemoryManagerTest { - - private static final long ZERO = System.currentTimeMillis(); - private static final long LATER = ZERO + 20 * 60 * 1000; - private static final long ONE_GIG = 1024 * 1024 * 1024; - private static final long HALF_GIG = ONE_GIG / 2; - private static final long QGIG = ONE_GIG / 4; - private static final long ONE_MINUTE = 60 * 1000; - - @Test - public void test() throws Exception { - LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest(); - ServerConfiguration config = new ServerConfiguration() { - ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance()); - - @Override - public AccumuloConfiguration getConfiguration() { - return DefaultConfiguration.getInstance(); - } - - @Override - public TableConfiguration getTableConfiguration(String tableId) { - return delegate.getTableConfiguration(tableId); - } - - @Override - public TableConfiguration getTableConfiguration(KeyExtent extent) { - return delegate.getTableConfiguration(extent); - } - - @Override - public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) { - return delegate.getNamespaceConfiguration(namespaceId); - } - - @Override - public Instance getInstance() { - return delegate.getInstance(); - } - }; - mgr.init(config); - MemoryManagementActions result; - // nothing to do - result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO, 2000, 0))); - assertEquals(0, result.tabletsToMinorCompact.size()); - // one tablet is really big - result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"), ZERO, 2000, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(k("x"), result.tabletsToMinorCompact.get(0)); - // one tablet is idle - mgr.currentTime = LATER; - result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, 2000, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(k("x"), result.tabletsToMinorCompact.get(0)); - // one tablet is idle, but one is really big - result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, ONE_GIG, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(k("y"), result.tabletsToMinorCompact.get(0)); - // lots of work to do - mgr = new LargestFirstMemoryManagerUnderTest(); - mgr.init(config); - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), - t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, HALF_GIG + 6, 0), - t(k("h"), ZERO, HALF_GIG + 7, 0), t(k("i"), ZERO, HALF_GIG + 8, 0))); - assertEquals(2, result.tabletsToMinorCompact.size()); - assertEquals(k("i"), result.tabletsToMinorCompact.get(0)); - assertEquals(k("h"), result.tabletsToMinorCompact.get(1)); - // one finished, one in progress, one filled up - mgr = new LargestFirstMemoryManagerUnderTest(); - mgr.init(config); - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), - t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG, 0), - t(k("h"), ZERO, 0, HALF_GIG + 7), t(k("i"), ZERO, 0, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(k("g"), result.tabletsToMinorCompact.get(0)); - // memory is very full, lots of candidates - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), - t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG + 6, 0), - t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0))); - assertEquals(2, result.tabletsToMinorCompact.size()); - assertEquals(k("g"), result.tabletsToMinorCompact.get(0)); - assertEquals(k("f"), result.tabletsToMinorCompact.get(1)); - // only have two compactors, still busy - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), - t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, ONE_GIG + 6), - t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0))); - assertEquals(0, result.tabletsToMinorCompact.size()); - // finished one - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), - t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, 0), - t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(k("e"), result.tabletsToMinorCompact.get(0)); - - // many are running: do nothing - mgr = new LargestFirstMemoryManagerUnderTest(); - mgr.init(config); - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), - t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"), ZERO, 0, HALF_GIG), t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG), - t(k("j"), ZERO, 0, HALF_GIG), t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG), t(k("m"), ZERO, 0, HALF_GIG))); - assertEquals(0, result.tabletsToMinorCompact.size()); - - // observe adjustment: - mgr = new LargestFirstMemoryManagerUnderTest(); - mgr.init(config); - // compact the largest - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(k("c"), result.tabletsToMinorCompact.get(0)); - // show that it is compacting... do nothing - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2))); - assertEquals(0, result.tabletsToMinorCompact.size()); - // not going to bother compacting any more - mgr.currentTime += ONE_MINUTE; - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2))); - assertEquals(0, result.tabletsToMinorCompact.size()); - // now do nothing - mgr.currentTime += ONE_MINUTE; - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, 0, 0), t(k("c"), ZERO, 0, 0))); - assertEquals(0, result.tabletsToMinorCompact.size()); - // on no! more data, this time we compact because we've adjusted - mgr.currentTime += ONE_MINUTE; - result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(k("b"), result.tabletsToMinorCompact.get(0)); - } - - @Test - public void testDeletedTable() throws Exception { - final String deletedTableId = "1"; - Function<String,Boolean> existenceCheck = new Function<String,Boolean>() { - public Boolean apply(String tableId) { - return !deletedTableId.equals(tableId); - } - }; - LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck); - ServerConfiguration config = new ServerConfiguration() { - ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance()); - - @Override - public AccumuloConfiguration getConfiguration() { - return DefaultConfiguration.getInstance(); - } - - @Override - public TableConfiguration getTableConfiguration(String tableId) { - return delegate.getTableConfiguration(tableId); - } - - @Override - public TableConfiguration getTableConfiguration(KeyExtent extent) { - return delegate.getTableConfiguration(extent); - } - - @Override - public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) { - return delegate.getNamespaceConfiguration(namespaceId); - } - - @Override - public Instance getInstance() { - return delegate.getInstance(); - } - }; - mgr.init(config); - MemoryManagementActions result; - // one tablet is really big and the other is for a nonexistent table - KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null); - result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0))); - assertEquals(1, result.tabletsToMinorCompact.size()); - assertEquals(extent, result.tabletsToMinorCompact.get(0)); - } - - private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager { - - public long currentTime = ZERO; - - @Override - protected long currentTimeMillis() { - return currentTime; - } - - @Override - protected long getMinCIdleThreshold(KeyExtent extent) { - return 15 * 60 * 1000; - } - - @Override - boolean tableExists(Instance instance, String tableId) { - return true; - } - } - - private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest { - - Function<String,Boolean> existenceCheck; - - public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck) { - super(); - this.existenceCheck = existenceCheck; - } - - @Override - boolean tableExists(Instance instance, String tableId) { - return existenceCheck.apply(tableId); - } - } - - private static KeyExtent k(String endRow) { - return new KeyExtent(new Text("1"), new Text(endRow), null); - } - - private static class TestTabletState implements TabletState { - - private final KeyExtent extent; - private final long lastCommit; - private final long memSize; - private final long compactingSize; - - TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize) { - this.extent = extent; - this.lastCommit = commit; - this.memSize = memsize; - this.compactingSize = compactingTableSize; - } - - @Override - public KeyExtent getExtent() { - return extent; - } - - @Override - public long getLastCommitTime() { - return lastCommit; - } - - @Override - public long getMemTableSize() { - return memSize; - } - - @Override - public long getMinorCompactingMemTableSize() { - return compactingSize; - } - - } - - private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize) { - return new TestTabletState(ke, lastCommit, memSize, compactingSize); - } - - private static List<TabletState> tablets(TabletState... states) { - return Arrays.asList(states); - } - -} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java ---------------------------------------------------------------------- diff --git a/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java new file mode 100644 index 0000000..f3bd220 --- /dev/null +++ b/server/tserver/src/test/java/org/apache/accumulo/tserver/LargestFirstMemoryManagerTest.java @@ -0,0 +1,298 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.accumulo.tserver; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.List; + +import org.apache.accumulo.core.client.Instance; +import org.apache.accumulo.core.client.mock.MockInstance; +import org.apache.accumulo.core.conf.AccumuloConfiguration; +import org.apache.accumulo.core.conf.DefaultConfiguration; +import org.apache.accumulo.core.data.impl.KeyExtent; +import org.apache.accumulo.server.conf.NamespaceConfiguration; +import org.apache.accumulo.server.conf.ServerConfiguration; +import org.apache.accumulo.server.conf.ServerConfigurationFactory; +import org.apache.accumulo.server.conf.TableConfiguration; +import org.apache.accumulo.server.tabletserver.LargestFirstMemoryManager; +import org.apache.accumulo.server.tabletserver.MemoryManagementActions; +import org.apache.accumulo.server.tabletserver.TabletState; +import org.apache.hadoop.io.Text; +import org.junit.Test; + +import com.google.common.base.Function; + +public class LargestFirstMemoryManagerTest { + + private static final long ZERO = System.currentTimeMillis(); + private static final long LATER = ZERO + 20 * 60 * 1000; + private static final long ONE_GIG = 1024 * 1024 * 1024; + private static final long HALF_GIG = ONE_GIG / 2; + private static final long QGIG = ONE_GIG / 4; + private static final long ONE_MINUTE = 60 * 1000; + + @Test + public void test() throws Exception { + LargestFirstMemoryManagerUnderTest mgr = new LargestFirstMemoryManagerUnderTest(); + ServerConfiguration config = new ServerConfiguration() { + ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance()); + + @Override + public AccumuloConfiguration getConfiguration() { + return DefaultConfiguration.getInstance(); + } + + @Override + public TableConfiguration getTableConfiguration(String tableId) { + return delegate.getTableConfiguration(tableId); + } + + @Override + public TableConfiguration getTableConfiguration(KeyExtent extent) { + return delegate.getTableConfiguration(extent); + } + + @Override + public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) { + return delegate.getNamespaceConfiguration(namespaceId); + } + + @Override + public Instance getInstance() { + return delegate.getInstance(); + } + }; + mgr.init(config); + MemoryManagementActions result; + // nothing to do + result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1000, 0), t(k("y"), ZERO, 2000, 0))); + assertEquals(0, result.tabletsToMinorCompact.size()); + // one tablet is really big + result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, ONE_GIG, 0), t(k("y"), ZERO, 2000, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(k("x"), result.tabletsToMinorCompact.get(0)); + // one tablet is idle + mgr.currentTime = LATER; + result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, 2000, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(k("x"), result.tabletsToMinorCompact.get(0)); + // one tablet is idle, but one is really big + result = mgr.getMemoryManagementActions(tablets(t(k("x"), ZERO, 1001, 0), t(k("y"), LATER, ONE_GIG, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(k("y"), result.tabletsToMinorCompact.get(0)); + // lots of work to do + mgr = new LargestFirstMemoryManagerUnderTest(); + mgr.init(config); + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), + t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, HALF_GIG + 6, 0), + t(k("h"), ZERO, HALF_GIG + 7, 0), t(k("i"), ZERO, HALF_GIG + 8, 0))); + assertEquals(2, result.tabletsToMinorCompact.size()); + assertEquals(k("i"), result.tabletsToMinorCompact.get(0)); + assertEquals(k("h"), result.tabletsToMinorCompact.get(1)); + // one finished, one in progress, one filled up + mgr = new LargestFirstMemoryManagerUnderTest(); + mgr.init(config); + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), + t(k("d"), ZERO, HALF_GIG + 3, 0), t(k("e"), ZERO, HALF_GIG + 4, 0), t(k("f"), ZERO, HALF_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG, 0), + t(k("h"), ZERO, 0, HALF_GIG + 7), t(k("i"), ZERO, 0, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(k("g"), result.tabletsToMinorCompact.get(0)); + // memory is very full, lots of candidates + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), + t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG + 5, 0), t(k("g"), ZERO, ONE_GIG + 6, 0), + t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0))); + assertEquals(2, result.tabletsToMinorCompact.size()); + assertEquals(k("g"), result.tabletsToMinorCompact.get(0)); + assertEquals(k("f"), result.tabletsToMinorCompact.get(1)); + // only have two compactors, still busy + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), + t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, ONE_GIG + 6), + t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0))); + assertEquals(0, result.tabletsToMinorCompact.size()); + // finished one + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, ONE_GIG + 1, 0), t(k("c"), ZERO, ONE_GIG + 2, 0), + t(k("d"), ZERO, ONE_GIG + 3, 0), t(k("e"), ZERO, ONE_GIG + 4, 0), t(k("f"), ZERO, ONE_GIG, ONE_GIG + 5), t(k("g"), ZERO, ONE_GIG, 0), + t(k("h"), ZERO, 0, 0), t(k("i"), ZERO, 0, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(k("e"), result.tabletsToMinorCompact.get(0)); + + // many are running: do nothing + mgr = new LargestFirstMemoryManagerUnderTest(); + mgr.init(config); + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, HALF_GIG, 0), t(k("b"), ZERO, HALF_GIG + 1, 0), t(k("c"), ZERO, HALF_GIG + 2, 0), + t(k("d"), ZERO, 0, HALF_GIG), t(k("e"), ZERO, 0, HALF_GIG), t(k("f"), ZERO, 0, HALF_GIG), t(k("g"), ZERO, 0, HALF_GIG), t(k("i"), ZERO, 0, HALF_GIG), + t(k("j"), ZERO, 0, HALF_GIG), t(k("k"), ZERO, 0, HALF_GIG), t(k("l"), ZERO, 0, HALF_GIG), t(k("m"), ZERO, 0, HALF_GIG))); + assertEquals(0, result.tabletsToMinorCompact.size()); + + // observe adjustment: + mgr = new LargestFirstMemoryManagerUnderTest(); + mgr.init(config); + // compact the largest + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, QGIG + 2, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(k("c"), result.tabletsToMinorCompact.get(0)); + // show that it is compacting... do nothing + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2))); + assertEquals(0, result.tabletsToMinorCompact.size()); + // not going to bother compacting any more + mgr.currentTime += ONE_MINUTE; + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, QGIG + 2))); + assertEquals(0, result.tabletsToMinorCompact.size()); + // now do nothing + mgr.currentTime += ONE_MINUTE; + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, 0, 0), t(k("c"), ZERO, 0, 0))); + assertEquals(0, result.tabletsToMinorCompact.size()); + // on no! more data, this time we compact because we've adjusted + mgr.currentTime += ONE_MINUTE; + result = mgr.getMemoryManagementActions(tablets(t(k("a"), ZERO, QGIG, 0), t(k("b"), ZERO, QGIG + 1, 0), t(k("c"), ZERO, 0, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(k("b"), result.tabletsToMinorCompact.get(0)); + } + + @Test + public void testDeletedTable() throws Exception { + final String deletedTableId = "1"; + Function<String,Boolean> existenceCheck = new Function<String,Boolean>() { + @Override + public Boolean apply(String tableId) { + return !deletedTableId.equals(tableId); + } + }; + LargestFirstMemoryManagerWithExistenceCheck mgr = new LargestFirstMemoryManagerWithExistenceCheck(existenceCheck); + ServerConfiguration config = new ServerConfiguration() { + ServerConfigurationFactory delegate = new ServerConfigurationFactory(new MockInstance()); + + @Override + public AccumuloConfiguration getConfiguration() { + return DefaultConfiguration.getInstance(); + } + + @Override + public TableConfiguration getTableConfiguration(String tableId) { + return delegate.getTableConfiguration(tableId); + } + + @Override + public TableConfiguration getTableConfiguration(KeyExtent extent) { + return delegate.getTableConfiguration(extent); + } + + @Override + public NamespaceConfiguration getNamespaceConfiguration(String namespaceId) { + return delegate.getNamespaceConfiguration(namespaceId); + } + + @Override + public Instance getInstance() { + return delegate.getInstance(); + } + }; + mgr.init(config); + MemoryManagementActions result; + // one tablet is really big and the other is for a nonexistent table + KeyExtent extent = new KeyExtent(new Text("2"), new Text("j"), null); + result = mgr.getMemoryManagementActions(tablets(t(extent, ZERO, ONE_GIG, 0), t(k("j"), ZERO, ONE_GIG, 0))); + assertEquals(1, result.tabletsToMinorCompact.size()); + assertEquals(extent, result.tabletsToMinorCompact.get(0)); + } + + private static class LargestFirstMemoryManagerUnderTest extends LargestFirstMemoryManager { + + public long currentTime = ZERO; + + @Override + protected long currentTimeMillis() { + return currentTime; + } + + @Override + protected long getMinCIdleThreshold(KeyExtent extent) { + return 15 * 60 * 1000; + } + + @Override + protected boolean tableExists(Instance instance, String tableId) { + return true; + } + } + + private static class LargestFirstMemoryManagerWithExistenceCheck extends LargestFirstMemoryManagerUnderTest { + + Function<String,Boolean> existenceCheck; + + public LargestFirstMemoryManagerWithExistenceCheck(Function<String,Boolean> existenceCheck) { + super(); + this.existenceCheck = existenceCheck; + } + + @Override + protected boolean tableExists(Instance instance, String tableId) { + return existenceCheck.apply(tableId); + } + } + + private static KeyExtent k(String endRow) { + return new KeyExtent(new Text("1"), new Text(endRow), null); + } + + private static class TestTabletState implements TabletState { + + private final KeyExtent extent; + private final long lastCommit; + private final long memSize; + private final long compactingSize; + + TestTabletState(KeyExtent extent, long commit, long memsize, long compactingTableSize) { + this.extent = extent; + this.lastCommit = commit; + this.memSize = memsize; + this.compactingSize = compactingTableSize; + } + + @Override + public KeyExtent getExtent() { + return extent; + } + + @Override + public long getLastCommitTime() { + return lastCommit; + } + + @Override + public long getMemTableSize() { + return memSize; + } + + @Override + public long getMinorCompactingMemTableSize() { + return compactingSize; + } + + } + + private TabletState t(KeyExtent ke, long lastCommit, long memSize, long compactingSize) { + return new TestTabletState(ke, lastCommit, memSize, compactingSize); + } + + private static List<TabletState> tablets(TabletState... states) { + return Arrays.asList(states); + } + +} http://git-wip-us.apache.org/repos/asf/accumulo/blob/d10840b8/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java ---------------------------------------------------------------------- diff --git a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java b/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java deleted file mode 100644 index b388cbf..0000000 --- a/test/src/test/java/org/apache/accumulo/proxy/ProxyDurabilityIT.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.accumulo.proxy; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -import java.io.File; -import java.nio.ByteBuffer; -import java.nio.file.Files; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.TreeMap; - -import org.apache.accumulo.core.client.Connector; -import org.apache.accumulo.core.client.security.tokens.PasswordToken; -import org.apache.accumulo.core.conf.Property; -import org.apache.accumulo.core.security.Authorizations; -import org.apache.accumulo.core.util.UtilWaitThread; -import org.apache.accumulo.minicluster.ServerType; -import org.apache.accumulo.minicluster.impl.MiniAccumuloConfigImpl; -import org.apache.accumulo.minicluster.impl.ProcessReference; -import org.apache.accumulo.proxy.thrift.AccumuloProxy.Client; -import org.apache.accumulo.proxy.thrift.Column; -import org.apache.accumulo.proxy.thrift.ColumnUpdate; -import org.apache.accumulo.proxy.thrift.Condition; -import org.apache.accumulo.proxy.thrift.ConditionalStatus; -import org.apache.accumulo.proxy.thrift.ConditionalUpdates; -import org.apache.accumulo.proxy.thrift.ConditionalWriterOptions; -import org.apache.accumulo.proxy.thrift.Durability; -import org.apache.accumulo.proxy.thrift.TimeType; -import org.apache.accumulo.proxy.thrift.WriterOptions; -import org.apache.accumulo.server.util.PortUtils; -import org.apache.accumulo.test.functional.ConfigurableMacIT; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.RawLocalFileSystem; -import org.apache.thrift.protocol.TJSONProtocol; -import org.apache.thrift.server.TServer; -import org.junit.Test; - -import com.google.common.collect.Iterators; -import com.google.common.net.HostAndPort; - -public class ProxyDurabilityIT extends ConfigurableMacIT { - - @Override - public void configure(MiniAccumuloConfigImpl cfg, Configuration hadoopCoreSite) { - hadoopCoreSite.set("fs.file.impl", RawLocalFileSystem.class.getName()); - cfg.setProperty(Property.INSTANCE_ZK_TIMEOUT, "10s"); - cfg.setNumTservers(1); - } - - private static ByteBuffer bytes(String value) { - return ByteBuffer.wrap(value.getBytes()); - } - - @Test - public void testDurability() throws Exception { - Connector c = getConnector(); - Properties props = new Properties(); - // Avoid issues with locally installed client configuration files with custom properties - File emptyFile = Files.createTempFile(null, null).toFile(); - emptyFile.deleteOnExit(); - props.put("instance", c.getInstance().getInstanceName()); - props.put("zookeepers", c.getInstance().getZooKeepers()); - props.put("tokenClass", PasswordToken.class.getName()); - props.put("clientConfigurationFile", emptyFile.toString()); - - TJSONProtocol.Factory protocol = new TJSONProtocol.Factory(); - - int proxyPort = PortUtils.getRandomFreePort(); - final TServer proxyServer = Proxy.createProxyServer(HostAndPort.fromParts("localhost", proxyPort), protocol, props).server; - while (!proxyServer.isServing()) - UtilWaitThread.sleep(100); - Client client = new TestProxyClient("localhost", proxyPort, protocol).proxy(); - Map<String,String> properties = new TreeMap<String,String>(); - properties.put("password", ROOT_PASSWORD); - ByteBuffer login = client.login("root", properties); - - String tableName = getUniqueNames(1)[0]; - client.createTable(login, tableName, true, TimeType.MILLIS); - assertTrue(c.tableOperations().exists(tableName)); - - WriterOptions options = new WriterOptions(); - options.setDurability(Durability.NONE); - String writer = client.createWriter(login, tableName, options); - Map<ByteBuffer,List<ColumnUpdate>> cells = new TreeMap<ByteBuffer,List<ColumnUpdate>>(); - ColumnUpdate column = new ColumnUpdate(bytes("cf"), bytes("cq")); - column.setValue("value".getBytes()); - cells.put(bytes("row"), Collections.singletonList(column)); - client.update(writer, cells); - client.closeWriter(writer); - assertEquals(1, count(tableName)); - restartTServer(); - assertEquals(0, count(tableName)); - - ConditionalWriterOptions cfg = new ConditionalWriterOptions(); - cfg.setDurability(Durability.SYNC); - String cwriter = client.createConditionalWriter(login, tableName, cfg); - ConditionalUpdates updates = new ConditionalUpdates(); - updates.addToConditions(new Condition(new Column(bytes("cf"), bytes("cq"), bytes("")))); - updates.addToUpdates(column); - Map<ByteBuffer,ConditionalStatus> status = client.updateRowsConditionally(cwriter, Collections.singletonMap(bytes("row"), updates)); - assertEquals(ConditionalStatus.ACCEPTED, status.get(bytes("row"))); - assertEquals(1, count(tableName)); - restartTServer(); - assertEquals(1, count(tableName)); - - proxyServer.stop(); - } - - private void restartTServer() throws Exception { - for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) { - cluster.killProcess(ServerType.TABLET_SERVER, proc); - } - cluster.start(); - } - - private int count(String tableName) throws Exception { - return Iterators.size((getConnector().createScanner(tableName, Authorizations.EMPTY)).iterator()); - } - -}
