This is an automated email from the ASF dual-hosted git repository. ctubbsii pushed a commit to branch temp-3142-merge in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/temp-3142-merge by this push: new c8fa164a79 Add a last location mode property (#3142) c8fa164a79 is described below commit c8fa164a792490b48d93588411c61c449197a967 Author: Ivan Bella <i...@bella.name> AuthorDate: Thu Jan 26 11:45:48 2023 -0500 Add a last location mode property (#3142) This adds a property to allow users to change when the last location is updated in a tablet's metadata. The last location can be updated on compaction or on assignment. In combination with the `manager.startup.tserver.*` properties, this will allow users to better manage initial assignments of tablets according to their preferences, depending on the configured balancer that respects the last location field. Co-authored-by: Christopher Tubbs <ctubb...@apache.org> --- .../org/apache/accumulo/core/conf/Property.java | 8 ++ .../apache/accumulo/core/conf/PropertyType.java | 3 + .../accumulo/core/conf/PropertyTypeTest.java | 6 ++ .../server/manager/state/MetaDataStateStore.java | 14 ++- .../server/manager/state/TabletStateStore.java | 2 +- .../server/manager/state/ZooTabletStateStore.java | 12 ++- .../accumulo/server/util/ManagerMetadataUtil.java | 79 +++++++++++--- .../manager/state/RootTabletStateStoreTest.java | 9 +- .../accumulo/manager/TabletGroupWatcher.java | 9 ++ .../test/functional/AssignLocationModeIT.java | 111 +++++++++++++++++++ .../test/functional/CompactLocationModeIT.java | 118 +++++++++++++++++++++ 11 files changed, 350 insertions(+), 21 deletions(-) diff --git a/core/src/main/java/org/apache/accumulo/core/conf/Property.java b/core/src/main/java/org/apache/accumulo/core/conf/Property.java index 3acd28d476..832327bbd8 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/Property.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/Property.java @@ -705,6 +705,14 @@ public enum Property { "The number of threads on each tablet server available to retrieve" + " summary data, that is not currently in cache, from RFiles.", "2.0.0"), + TSERV_LAST_LOCATION_MODE("tserver.last.location.mode", "compaction", + PropertyType.LAST_LOCATION_MODE, + "Describes how the system will record the 'last' location for tablets, which can be used for assigning them when a cluster restarts." + + " If 'compaction' is the mode, then the system will record the location where the tablet's most recent compaction occurred." + + " If 'assignment' is the mode, then the most recently assigned location will be recorded." + + " The manager.startup.tserver properties might also need to be set to ensure" + + " the tserver is available before tablets are initially assigned if the 'last' location is to be used.", + "3.0.0"), // accumulo garbage collector properties GC_PREFIX("gc.", null, PropertyType.PREFIX, diff --git a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java index c590054485..b13a026c8c 100644 --- a/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java +++ b/core/src/main/java/org/apache/accumulo/core/conf/PropertyType.java @@ -128,6 +128,9 @@ public enum PropertyType { GC_POST_ACTION("gc_post_action", in(true, null, "none", "flush", "compact"), "One of 'none', 'flush', or 'compact'."), + LAST_LOCATION_MODE("last_location_mode", in(true, null, "assignment", "compaction"), + "Defines how to update the last location. One of 'assignment', or 'compaction'."), + STRING("string", x -> true, "An arbitrary string of characters whose format is unspecified and" + " interpreted based on the context of the property to which it applies."), diff --git a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java index 412a67180f..514fd1e95c 100644 --- a/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java +++ b/core/src/test/java/org/apache/accumulo/core/conf/PropertyTypeTest.java @@ -144,6 +144,12 @@ public class PropertyTypeTest extends WithTestNames { invalid("", "other"); } + @Test + public void testTypeLAST_LOCATION_MODE() { + valid(null, "compaction", "assignment"); + invalid("", "other"); + } + @Test public void testTypeFRACTION() { valid(null, "1", "0", "1.0", "25%", "2.5%", "10.2E-3", "10.2E-3%", ".3"); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java index 9cf186d89a..18a3386360 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/MetaDataStateStore.java @@ -31,6 +31,7 @@ import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator; import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.util.ManagerMetadataUtil; import org.apache.hadoop.fs.Path; class MetaDataStateStore implements TabletStateStore { @@ -60,9 +61,13 @@ class MetaDataStateStore implements TabletStateStore { public void setLocations(Collection<Assignment> assignments) throws DistributedStoreException { try (var tabletsMutator = ample.mutateTablets()) { for (Assignment assignment : assignments) { - tabletsMutator.mutateTablet(assignment.tablet) - .putLocation(assignment.server, LocationType.CURRENT) - .deleteLocation(assignment.server, LocationType.FUTURE).deleteSuspension().mutate(); + TabletMutator tabletMutator = tabletsMutator.mutateTablet(assignment.tablet); + tabletMutator.putLocation(assignment.server, LocationType.CURRENT); + ManagerMetadataUtil.updateLastForAssignmentMode(context, ample, tabletMutator, + assignment.tablet, assignment.server); + tabletMutator.deleteLocation(assignment.server, LocationType.FUTURE); + tabletMutator.deleteSuspension(); + tabletMutator.mutate(); } } catch (RuntimeException ex) { throw new DistributedStoreException(ex); @@ -102,6 +107,8 @@ class MetaDataStateStore implements TabletStateStore { for (TabletLocationState tls : tablets) { TabletMutator tabletMutator = tabletsMutator.mutateTablet(tls.extent); if (tls.current != null) { + ManagerMetadataUtil.updateLastForAssignmentMode(context, ample, tabletMutator, tls.extent, + tls.current); tabletMutator.deleteLocation(tls.current, LocationType.CURRENT); if (logsForDeadServers != null) { List<Path> logs = logsForDeadServers.get(tls.current); @@ -147,4 +154,5 @@ class MetaDataStateStore implements TabletStateStore { public String name() { return "Normal Tablets"; } + } diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java index 999667f9b8..c96912ee58 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/TabletStateStore.java @@ -112,7 +112,7 @@ public interface TabletStateStore extends Iterable<TabletLocationState> { TabletStateStore tss; switch (level) { case ROOT: - tss = new ZooTabletStateStore(context.getAmple()); + tss = new ZooTabletStateStore(context); break; case METADATA: tss = new RootTabletStateStore(context, state); diff --git a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java index e412b09837..4cee8f8e23 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java +++ b/server/base/src/main/java/org/apache/accumulo/server/manager/state/ZooTabletStateStore.java @@ -24,6 +24,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.metadata.RootTable; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletLocationState; @@ -34,6 +35,7 @@ import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.Location; import org.apache.accumulo.core.metadata.schema.TabletMetadata.LocationType; import org.apache.accumulo.core.tabletserver.log.LogEntry; +import org.apache.accumulo.server.util.ManagerMetadataUtil; import org.apache.hadoop.fs.Path; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,9 +44,11 @@ class ZooTabletStateStore implements TabletStateStore { private static final Logger log = LoggerFactory.getLogger(ZooTabletStateStore.class); private final Ample ample; + private final ClientContext context; - ZooTabletStateStore(Ample ample) { - this.ample = ample; + ZooTabletStateStore(ClientContext context) { + this.context = context; + this.ample = context.getAmple(); } @Override @@ -134,6 +138,8 @@ class ZooTabletStateStore implements TabletStateStore { TabletMutator tabletMutator = ample.mutateTablet(assignment.tablet); tabletMutator.putLocation(assignment.server, LocationType.CURRENT); + ManagerMetadataUtil.updateLastForAssignmentMode(context, ample, tabletMutator, + assignment.tablet, assignment.server); tabletMutator.deleteLocation(assignment.server, LocationType.FUTURE); tabletMutator.mutate(); @@ -154,6 +160,8 @@ class ZooTabletStateStore implements TabletStateStore { tabletMutator.deleteLocation(tls.futureOrCurrent(), LocationType.FUTURE); tabletMutator.deleteLocation(tls.futureOrCurrent(), LocationType.CURRENT); + ManagerMetadataUtil.updateLastForAssignmentMode(context, ample, tabletMutator, tls.extent, + tls.futureOrCurrent()); if (logsForDeadServers != null) { List<Path> logs = logsForDeadServers.get(tls.futureOrCurrent()); if (logs != null) { diff --git a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java index ea54864a7a..6652d59f07 100644 --- a/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java +++ b/server/base/src/main/java/org/apache/accumulo/server/util/ManagerMetadataUtil.java @@ -34,7 +34,9 @@ import java.util.concurrent.TimeUnit; import org.apache.accumulo.core.client.AccumuloException; import org.apache.accumulo.core.client.Scanner; +import org.apache.accumulo.core.clientImpl.ClientContext; import org.apache.accumulo.core.clientImpl.ScannerImpl; +import org.apache.accumulo.core.conf.Property; import org.apache.accumulo.core.data.Key; import org.apache.accumulo.core.data.PartialKey; import org.apache.accumulo.core.data.Range; @@ -46,6 +48,7 @@ import org.apache.accumulo.core.metadata.MetadataTable; import org.apache.accumulo.core.metadata.StoredTabletFile; import org.apache.accumulo.core.metadata.TServerInstance; import org.apache.accumulo.core.metadata.TabletFile; +import org.apache.accumulo.core.metadata.schema.Ample; import org.apache.accumulo.core.metadata.schema.Ample.TabletMutator; import org.apache.accumulo.core.metadata.schema.DataFileValue; import org.apache.accumulo.core.metadata.schema.ExternalCompactionId; @@ -204,13 +207,7 @@ public class ManagerMetadataUtil { tablet.putCompactionId(compactionId); } - TServerInstance self = getTServerInstance(address, zooLock); - tablet.putLocation(self, LocationType.LAST); - - // remove the old location - if (lastLocation != null && !lastLocation.equals(self)) { - tablet.deleteLocation(lastLocation, LocationType.LAST); - } + updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock); if (ecid.isPresent()) { tablet.deleteExternalCompaction(ecid.get()); @@ -239,13 +236,7 @@ public class ManagerMetadataUtil { tablet.putTime(time); newFile = Optional.of(newDatafile.insert()); - TServerInstance self = getTServerInstance(address, zooLock); - tablet.putLocation(self, LocationType.LAST); - - // remove the old location - if (lastLocation != null && !lastLocation.equals(self)) { - tablet.deleteLocation(lastLocation, LocationType.LAST); - } + updateLastForCompactionMode(context, tablet, lastLocation, address, zooLock); } tablet.putFlushId(flushId); @@ -256,4 +247,64 @@ public class ManagerMetadataUtil { tablet.mutate(); return newFile; } + + /** + * Update the last location if the location mode is "assignment". This will delete the previous + * last location if needed and set the new last location + * + * @param context The server context + * @param ample The metadata persistence layer + * @param tabletMutator The mutator being built + * @param extent The tablet extent + * @param location The new location + */ + public static void updateLastForAssignmentMode(ClientContext context, Ample ample, + Ample.TabletMutator tabletMutator, KeyExtent extent, TServerInstance location) { + // if the location mode is assignment, then preserve the current location in the last + // location value + if ("assignment".equals(context.getConfiguration().get(Property.TSERV_LAST_LOCATION_MODE))) { + TabletMetadata lastMetadata = ample.readTablet(extent, TabletMetadata.ColumnType.LAST); + TServerInstance lastLocation = (lastMetadata == null ? null : lastMetadata.getLast()); + ManagerMetadataUtil.updateLast(tabletMutator, lastLocation, location); + } + } + + /** + * Update the last location if the location mode is "compaction". This will delete the previous + * last location if needed and set the new last location + * + * @param context The server context + * @param tabletMutator The mutator being built + * @param lastLocation The last location + * @param address The server address + * @param zooLock The zookeeper lock + */ + public static void updateLastForCompactionMode(ClientContext context, TabletMutator tabletMutator, + TServerInstance lastLocation, String address, ServiceLock zooLock) { + // if the location mode is 'compaction', then preserve the current compaction location in the + // last location value + if ("compaction".equals(context.getConfiguration().get(Property.TSERV_LAST_LOCATION_MODE))) { + TServerInstance newLocation = getTServerInstance(address, zooLock); + updateLast(tabletMutator, lastLocation, newLocation); + } + } + + /** + * Update the last location, deleting the previous location if needed + * + * @param tabletMutator The mutator being built + * @param lastLocation The last location (may be null) + * @param newLocation The new location + */ + public static void updateLast(TabletMutator tabletMutator, TServerInstance lastLocation, + TServerInstance newLocation) { + if (lastLocation != null) { + if (!lastLocation.equals(newLocation)) { + tabletMutator.deleteLocation(lastLocation, LocationType.LAST); + tabletMutator.putLocation(newLocation, LocationType.LAST); + } + } else { + tabletMutator.putLocation(newLocation, LocationType.LAST); + } + } } diff --git a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java index 86e8eb96bc..90079a9bfc 100644 --- a/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java +++ b/server/base/src/test/java/org/apache/accumulo/server/manager/state/RootTabletStateStoreTest.java @@ -20,6 +20,7 @@ package org.apache.accumulo.server.manager.state; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.accumulo.server.init.ZooKeeperInitializer.getInitialRootTabletJson; +import static org.easymock.EasyMock.expect; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertThrows; @@ -40,7 +41,10 @@ import org.apache.accumulo.core.metadata.schema.RootTabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata; import org.apache.accumulo.core.metadata.schema.TabletMetadata.ColumnType; import org.apache.accumulo.core.metadata.schema.TabletsMetadata; +import org.apache.accumulo.server.MockServerContext; +import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.metadata.TabletMutatorBase; +import org.easymock.EasyMock; import org.junit.jupiter.api.Test; import com.google.common.base.Preconditions; @@ -85,7 +89,10 @@ public class RootTabletStateStoreTest { @Test public void testRootTabletStateStore() throws DistributedStoreException { - ZooTabletStateStore tstore = new ZooTabletStateStore(new TestAmple()); + ServerContext context = MockServerContext.get(); + expect(context.getAmple()).andReturn(new TestAmple()).anyTimes(); + EasyMock.replay(context); + ZooTabletStateStore tstore = new ZooTabletStateStore(context); KeyExtent root = RootTable.EXTENT; String sessionId = "this is my unique session data"; TServerInstance server = diff --git a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java index 30c4e4ee94..33c4346dff 100644 --- a/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java +++ b/server/manager/src/main/java/org/apache/accumulo/manager/TabletGroupWatcher.java @@ -915,6 +915,15 @@ abstract class TabletGroupWatcher extends AccumuloDaemonThread { continue; } + TServerInstance lastLocation = unassigned.get(assignment.getKey()); + if (lastLocation != null + && !assignment.getValue().getHostPort().equals(lastLocation.getHostPort())) { + Manager.log.warn( + "balancer assigned {} to {} which is not the suggested location of {}", + assignment.getKey(), assignment.getValue().getHostPort(), + lastLocation.getHostPort()); + } + tLists.assignments.add(new Assignment(assignment.getKey(), assignment.getValue())); } } else { diff --git a/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java new file mode 100644 index 0000000000..dae97130f7 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/AssignLocationModeIT.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.functional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.time.Duration; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.TabletLocationState; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.manager.state.MetaDataTableScanner; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +public class AssignLocationModeIT extends ConfigurableMacBase { + + @Override + protected Duration defaultTimeout() { + return Duration.ofMinutes(2); + } + + @Override + public void configure(MiniAccumuloConfigImpl cfg, Configuration fsConf) { + cfg.setProperty(Property.TSERV_LAST_LOCATION_MODE, "assignment"); + } + + @Test + public void test() throws Exception { + try (AccumuloClient c = + getCluster().createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) { + String tableName = super.getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + String tableId = c.tableOperations().tableIdMap().get(tableName); + // wait for the table to be online + TabletLocationState newTablet; + do { + UtilWaitThread.sleep(250); + newTablet = getTabletLocationState(c, tableId); + } while (newTablet.current == null); + // this would be null if the mode was not "assign" + assertEquals(newTablet.current, newTablet.last); + assertNull(newTablet.future); + + // put something in it + try (BatchWriter bw = c.createBatchWriter(tableName)) { + Mutation m = new Mutation("a"); + m.put("b", "c", "d"); + bw.addMutation(m); + } + // assert that the default mode is "assign" + assertEquals("assignment", c.instanceOperations().getSystemConfiguration() + .get(Property.TSERV_LAST_LOCATION_MODE.getKey())); + + // last location should not be set yet + TabletLocationState unflushed = getTabletLocationState(c, tableId); + assertEquals(newTablet.current, unflushed.current); + assertEquals(newTablet.current, unflushed.last); + assertNull(newTablet.future); + + // take the tablet offline + c.tableOperations().offline(tableName, true); + TabletLocationState offline = getTabletLocationState(c, tableId); + assertNull(offline.future); + assertNull(offline.current); + assertEquals(newTablet.current, offline.last); + + // put it back online, should have the same last location + c.tableOperations().online(tableName, true); + TabletLocationState online = getTabletLocationState(c, tableId); + assertNull(online.future); + assertNotNull(online.current); + assertEquals(newTablet.last, online.last); + } + } + + private TabletLocationState getTabletLocationState(AccumuloClient c, String tableId) { + try (MetaDataTableScanner s = new MetaDataTableScanner((ClientContext) c, + new Range(TabletsSection.encodeRow(TableId.of(tableId), null)), MetadataTable.NAME)) { + return s.next(); + } + } +} diff --git a/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java new file mode 100644 index 0000000000..4f2dccaa97 --- /dev/null +++ b/test/src/main/java/org/apache/accumulo/test/functional/CompactLocationModeIT.java @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.accumulo.test.functional; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; + +import java.time.Duration; + +import org.apache.accumulo.core.client.AccumuloClient; +import org.apache.accumulo.core.client.BatchWriter; +import org.apache.accumulo.core.client.security.tokens.PasswordToken; +import org.apache.accumulo.core.clientImpl.ClientContext; +import org.apache.accumulo.core.conf.Property; +import org.apache.accumulo.core.data.Mutation; +import org.apache.accumulo.core.data.Range; +import org.apache.accumulo.core.data.TableId; +import org.apache.accumulo.core.metadata.MetadataTable; +import org.apache.accumulo.core.metadata.TabletLocationState; +import org.apache.accumulo.core.metadata.schema.MetadataSchema.TabletsSection; +import org.apache.accumulo.core.util.UtilWaitThread; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; +import org.apache.accumulo.server.manager.state.MetaDataTableScanner; +import org.apache.hadoop.conf.Configuration; +import org.junit.jupiter.api.Test; + +public class CompactLocationModeIT extends ConfigurableMacBase { + + @Override + protected Duration defaultTimeout() { + return Duration.ofMinutes(2); + } + + @Override + public void configure(MiniAccumuloConfigImpl cfg, Configuration fsConf) { + cfg.setProperty(Property.TSERV_LAST_LOCATION_MODE, "compaction"); + } + + @Test + public void test() throws Exception { + try (AccumuloClient c = + getCluster().createAccumuloClient("root", new PasswordToken(ROOT_PASSWORD))) { + String tableName = super.getUniqueNames(1)[0]; + c.tableOperations().create(tableName); + String tableId = c.tableOperations().tableIdMap().get(tableName); + // wait for the table to be online + TabletLocationState newTablet; + do { + UtilWaitThread.sleep(250); + newTablet = getTabletLocationState(c, tableId); + } while (newTablet.current == null); + assertNull(newTablet.last); + assertNull(newTablet.future); + + // put something in it + try (BatchWriter bw = c.createBatchWriter(tableName)) { + Mutation m = new Mutation("a"); + m.put("b", "c", "d"); + bw.addMutation(m); + } + // assert that the default mode is "compact" + assertEquals("compaction", c.instanceOperations().getSystemConfiguration() + .get(Property.TSERV_LAST_LOCATION_MODE.getKey())); + + // no last location should be set yet + TabletLocationState unflushed = getTabletLocationState(c, tableId); + assertEquals(newTablet.current, unflushed.current); + assertNull(unflushed.last); + assertNull(newTablet.future); + + // This should give it a last location if the mode is being used correctly + c.tableOperations().flush(tableName, null, null, true); + + TabletLocationState flushed = getTabletLocationState(c, tableId); + assertEquals(newTablet.current, flushed.current); + assertEquals(flushed.current, flushed.last); + assertNull(newTablet.future); + + // take the tablet offline + c.tableOperations().offline(tableName, true); + TabletLocationState offline = getTabletLocationState(c, tableId); + assertNull(offline.future); + assertNull(offline.current); + assertEquals(flushed.current, offline.last); + + // put it back online, should have the same last location + c.tableOperations().online(tableName, true); + TabletLocationState online = getTabletLocationState(c, tableId); + assertNull(online.future); + assertNotNull(online.current); + assertEquals(offline.last, online.last); + } + } + + private TabletLocationState getTabletLocationState(AccumuloClient c, String tableId) { + try (MetaDataTableScanner s = new MetaDataTableScanner((ClientContext) c, + new Range(TabletsSection.encodeRow(TableId.of(tableId), null)), MetadataTable.NAME)) { + return s.next(); + } + } +}